💬 Natural Language Processing (NLP)

import nltk
from nltk import word_tokenize

# Download the tokenizer data (only needed once)
nltk.download('punkt')

# Our sentence
sentence = "Elon Musk founded SpaceX in California in 2002."

# Tokenize - break into individual words
tokens = word_tokenize(sentence)

print("Original sentence:", sentence)
print("Tokens:", tokens)
print("Number of tokens:", len(tokens))

# Output:
# Original sentence: Elon Musk founded SpaceX in California in 2002.
# Tokens: ['Elon', 'Musk', 'founded', 'SpaceX', 'in', 'California', 'in', '2002', '.']
# Number of tokens: 9

import nltk
from nltk import word_tokenize, pos_tag

# Download necessary data
nltk.download('averaged_perceptron_tagger')

sentence = "Elon Musk founded SpaceX in California in 2002."

# Step 1: Tokenize
tokens = word_tokenize(sentence)

# Step 2: POS Tag each token
tagged = pos_tag(tokens)

print("Word → Part of Speech:")
for word, tag in tagged:
    print(f"  {word} → {tag}")

# Output:
# Word → Part of Speech:
#   Elon → NNP (Proper Noun)
#   Musk → NNP (Proper Noun)
#   founded → VBD (Verb, Past Tense)
#   SpaceX → NNP (Proper Noun)
#   in → IN (Preposition)
#   California → NNP (Proper Noun)
#   in → IN (Preposition)
#   2002 → CD (Cardinal Number)
#   . → . (Punctuation)

import nltk
from nltk import word_tokenize, pos_tag, ne_chunk

# Download necessary data
nltk.download('maxent_ne_chunker')
nltk.download('words')

sentence = "Elon Musk founded SpaceX in California in 2002."

# Step 1: Tokenize
tokens = word_tokenize(sentence)

# Step 2: POS Tag
tagged = pos_tag(tokens)

# Step 3: Named Entity Recognition
entities = ne_chunk(tagged)

print("Named Entities Found:")
print(entities)

# Output:
# (S
#   (PERSON Elon/NNP)      ← Person's name!
#   (PERSON Musk/NNP)      ← Person's name!
#   founded/VBD
#   (ORGANIZATION SpaceX/NNP)  ← Company!
#   in/IN
#   (GPE California/NNP)   ← Place!
#   in/IN
#   2002/CD
#   ./.)

from sklearn.feature_extraction.text import TfidfVectorizer

# Our 3 documents
docs = [
    "Data Science is fun",
    "Python makes Data Analysis easy",
    "AI and Data Science are related"
]

# Create the TF-IDF vectorizer
vectorizer = TfidfVectorizer()

# Calculate TF-IDF for each word in each document
tfidf_matrix = vectorizer.fit_transform(docs)

# See the vocabulary (unique words)
print("Words found:")
print(vectorizer.get_feature_names_out())

# Output: ['ai', 'analysis', 'and', 'are', 'data', 'easy', 'fun', 
#          'is', 'makes', 'python', 'related', 'science']

# See the TF-IDF scores
print("\nTF-IDF Matrix (rows=docs, cols=words):")
print(tfidf_matrix.toarray().round(2))

# Note: Higher numbers = more important in that document!
# "data" has low scores (common), "python" has high score (unique)

# Remove common words like "is", "and", "the"
vectorizer = TfidfVectorizer(stop_words='english')
tfidf_matrix = vectorizer.fit_transform(docs)

print("Words after removing stop words:")
print(vectorizer.get_feature_names_out())

# Output: ['ai', 'analysis', 'data', 'easy', 'fun', 
#          'makes', 'python', 'related', 'science']
# Notice: 'is', 'and', 'are' are removed!

from nltk.stem import PorterStemmer, WordNetLemmatizer
from nltk.tokenize import word_tokenize
import nltk

nltk.download('wordnet')

sentence = "The children are playing happily while their teacher watches them."

# Create stemmer and lemmatizer
stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()

# Tokenize the sentence
tokens = word_tokenize(sentence)

# Apply stemming and lemmatization to each token
stems = [stemmer.stem(word) for word in tokens]
lemmas = [lemmatizer.lemmatize(word) for word in tokens]

print("Original:", tokens)
print("Stems:", stems)
print("Lemmas:", lemmas)

# Output:
# Original: ['The', 'children', 'are', 'playing', 'happily', 'while', 'their', 'teacher', 'watches', 'them', '.']
# Stems:    ['the', 'children', 'are', 'play', 'happili', 'while', 'their', 'teacher', 'watch', 'them', '.']
#           Note: 'happili' is not a real word! ❌
# Lemmas:   ['The', 'child', 'are', 'playing', 'happily', 'while', 'their', 'teacher', 'watch', 'them', '.']
#           Note: 'children' → 'child' (proper!) ✅