Sign in to save your place.
π APIs & SQL Databases
Get real data from the web and databases
πΊοΈ What You'll Learn
Run this on the page
Session_12 Read data from API and SQL has requests, JSON, and SQL examplesβsame ideas. Run the example on this page. Set a timeout on every requests call.
Call the API like this
requests.get(url) with no timeout can hang. Check the status before you parse JSON.
import requests
response = requests.get("https://api.github.com/users/python", timeout=10)
response.raise_for_status()
payload = response.json()
print(payload["login"])π What is an API?
π‘ Application Programming Interface
π½οΈ API = Restaurant Waiter
You (Client) Waiter (API) Kitchen (Server)
βββββββββββββ ββββββββββββ βββββββββββββββββ
π π§βπ³ π¨βπ³
"I want pizza" β Takes order β Prepares pizza
π π§βπ³ π¨βπ³
Receives pizza β Brings food β Sends pizza
The waiter (API) is the middleman that:
β’ Takes your request
β’ Communicates with the kitchen
β’ Returns the result to you
- Get weather data from weather services
- Access social media data
- Retrieve stock prices
- Use AI/ML services
- Send emails, SMS
π‘ Making HTTP Requests
π The requests Library
# HTTP requests from Session 12
import requests
# Make a GET request
url = "https://api.github.com/users/python"
response = requests.get(url)
# Check status code
print("Status Code:", response.status_code)
print("Success?", response.status_code == 200)
# Common status codes:
# 200 = Success
# 404 = Not Found
# 401 = Unauthorized
# 500 = Server Error
Status Code: 200 Success? True
π¦ Working with JSON
ποΈ JSON = JavaScript Object Notation
JSON is the most common data format for APIs. It's similar to Python dictionaries!
# Working with JSON from Session 12
import requests
import json
# Get JSON data from API
url = "https://api.github.com/users/python"
response = requests.get(url)
# Parse JSON response
data = response.json() # Converts JSON to Python dict
print("Type:", type(data))
print("Login:", data['login'])
print("Name:", data.get('name'))
print("Followers:", data.get('followers'))
# Working with JSON strings
json_string = '{"name": "Alice", "age": 25}'
parsed = json.loads(json_string) # String to dict
print("\\nParsed:", parsed)
# Convert dict to JSON string
my_dict = {"product": "Laptop", "price": 999}
json_str = json.dumps(my_dict, indent=2)
print("\\nJSON string:")
print(json_str)
Type: <class 'dict'>
Login: python
Name: Python
Followers: 17000+
Parsed: {'name': 'Alice', 'age': 25}
JSON string:
{
"product": "Laptop",
"price": 999
}
π€οΈ Practical Example: Weather API
# API example from Session 12
import requests
import pandas as pd
# Example: JSONPlaceholder API (free test API)
url = "https://jsonplaceholder.typicode.com/users"
response = requests.get(url)
users = response.json()
# Convert to DataFrame
df = pd.DataFrame(users)
print("Users from API:")
print(df[['id', 'name', 'email']].head())
# Get todos from API
todos_url = "https://jsonplaceholder.typicode.com/todos"
todos = requests.get(todos_url).json()
todos_df = pd.DataFrame(todos)
print("\\nTodos:")
print(todos_df.head())
# Analyze the data
print("\\nCompleted tasks by user:")
print(todos_df.groupby('userId')['completed'].sum().head())
Users from API: id name email 0 1 Leanne Graham Sincere@april.biz 1 2 Ervin Howell Shanna@melissa.tv 2 3 Clementine Bauch Nathan@yesenia.net 3 4 Patricia Lebsack Julianne.OConner@kory.org 4 5 Chelsey Dietrich Lucio_Hettinger@annie.ca Todos: userId id title completed 0 1 1 delectus aut autem False 1 1 2 quis ut nam facilis... False 2 1 3 fugiat veniam minus False 3 1 4 et porro tempora True 4 1 5 laboriosam mollitia... False Completed tasks by user: userId 1 11 2 8 3 7 4 6 5 12 Name: completed, dtype: int64
ποΈ SQL Databases
πΎ SQLite - A File-Based Database
SQLite is a lightweight database that stores everything in a single file. Perfect for learning and small projects!
# SQLite basics from Session 12
import sqlite3
# Connect to database (creates if doesn't exist)
conn = sqlite3.connect(':memory:') # In-memory database
cursor = conn.cursor()
# Create a table
cursor.execute('''
CREATE TABLE employees (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
department TEXT,
salary REAL
)
''')
# Insert data
cursor.execute("INSERT INTO employees (name, department, salary) VALUES (?, ?, ?)",
('Alice', 'HR', 50000))
cursor.execute("INSERT INTO employees (name, department, salary) VALUES (?, ?, ?)",
('Bob', 'IT', 60000))
cursor.execute("INSERT INTO employees (name, department, salary) VALUES (?, ?, ?)",
('Charlie', 'IT', 70000))
# Commit changes
conn.commit()
# Query data
cursor.execute("SELECT * FROM employees")
results = cursor.fetchall()
print("All employees:")
for row in results:
print(row)
# Close connection
conn.close()
All employees: (1, 'Alice', 'HR', 50000.0) (2, 'Bob', 'IT', 60000.0) (3, 'Charlie', 'IT', 70000.0)
π Common SQL Queries
# SQL queries from Session 12
import sqlite3
conn = sqlite3.connect(':memory:')
cursor = conn.cursor()
# Create and populate table
cursor.execute('CREATE TABLE products (id INTEGER, name TEXT, price REAL, stock INTEGER)')
cursor.executemany('INSERT INTO products VALUES (?, ?, ?, ?)', [
(1, 'Laptop', 999.99, 50),
(2, 'Mouse', 29.99, 200),
(3, 'Keyboard', 79.99, 150),
(4, 'Monitor', 299.99, 75),
(5, 'USB Cable', 9.99, 500)
])
conn.commit()
# SELECT with WHERE
print("Products over $50:")
cursor.execute("SELECT * FROM products WHERE price > 50")
for row in cursor.fetchall():
print(row)
# ORDER BY
print("\\nProducts by price (descending):")
cursor.execute("SELECT name, price FROM products ORDER BY price DESC")
for row in cursor.fetchall():
print(row)
# Aggregate functions
cursor.execute("SELECT AVG(price), SUM(stock), COUNT(*) FROM products")
avg_price, total_stock, count = cursor.fetchone()
print(f"\\nAverage price: ${avg_price:.2f}")
print(f"Total stock: {total_stock}")
print(f"Product count: {count}")
conn.close()
Products over $50:
(1, 'Laptop', 999.99, 50)
(3, 'Keyboard', 79.99, 150)
(4, 'Monitor', 299.99, 75)
Products by price (descending):
('Laptop', 999.99)
('Monitor', 299.99)
('Keyboard', 79.99)
('Mouse', 29.99)
('USB Cable', 9.99)
Average price: $283.99
Total stock: 975
Product count: 5
πΌ Pandas + SQL
π Reading SQL into DataFrames
# Pandas + SQL from Session 12
import sqlite3
import pandas as pd
# Create database with data
conn = sqlite3.connect(':memory:')
cursor = conn.cursor()
cursor.execute('CREATE TABLE sales (id INTEGER, product TEXT, amount REAL, date TEXT)')
cursor.executemany('INSERT INTO sales VALUES (?, ?, ?, ?)', [
(1, 'Laptop', 999.99, '2024-01-15'),
(2, 'Mouse', 29.99, '2024-01-15'),
(3, 'Laptop', 999.99, '2024-01-16'),
(4, 'Keyboard', 79.99, '2024-01-16'),
(5, 'Mouse', 29.99, '2024-01-17')
])
conn.commit()
# Read SQL query into DataFrame
df = pd.read_sql_query("SELECT * FROM sales", conn)
print("Sales DataFrame:")
print(df)
# Use Pandas for analysis
print("\\nTotal sales by product:")
print(df.groupby('product')['amount'].sum())
# Write DataFrame to SQL
new_data = pd.DataFrame({
'name': ['Alice', 'Bob'],
'score': [95, 87]
})
new_data.to_sql('scores', conn, index=False)
print("\\nCreated scores table from DataFrame!")
conn.close()
Sales DataFrame: id product amount date 0 1 Laptop 999.99 2024-01-15 1 2 Mouse 29.99 2024-01-15 2 3 Laptop 999.99 2024-01-16 3 4 Keyboard 79.99 2024-01-16 4 5 Mouse 29.99 2024-01-17 Total sales by product: product Keyboard 79.99 Laptop 1999.98 Mouse 59.98 Name: amount, dtype: float64 Created scores table from DataFrame!
π Quick Reference
π API Functions
| Code | Description |
|---|---|
| requests.get(url) | Make GET request |
| response.json() | Parse JSON response |
| response.status_code | HTTP status code |
| json.loads(str) | String to dict |
| json.dumps(dict) | Dict to string |
ποΈ SQL Functions
| Code | Description |
|---|---|
| sqlite3.connect() | Connect to database |
| cursor.execute() | Run SQL query |
| cursor.fetchall() | Get all results |
| conn.commit() | Save changes |
| pd.read_sql_query() | SQL to DataFrame |
| df.to_sql() | DataFrame to SQL |
π Congratulations!
You've finished session 12 of 15. Data visualization is next, then a second visualization session, then web scraping.
You now know:
- β Python fundamentals
- β Data types and structures
- β Control flow and loops
- β Functions and OOP
- β NumPy and Pandas
- β APIs and databases
π« Common Mistakes (APIs & SQL)
- Hardcoding API keys or passwords β Use environment variables (
os.environ.get("API_KEY")); never commit secrets to git. - SQL injection β Don't build queries with f-strings like
f"SELECT * FROM users WHERE id = {user_id}"; use parameterized queries (e.g.cursor.execute("... WHERE id = ?", (user_id,))). - Not checking response status β Check
response.status_codeor handle exceptions; 4xx/5xx mean the request failed.
π Short reflection
In one sentence: why should you never put API keys or database passwords directly in your source code?
β CORE (Must know)
- APIs: requests or SDK; auth (e.g. API key in env); parse JSON response.
- SQL: connect (e.g. sqlite3, psycopg2); execute queries; fetch results; parameterize to avoid injection.
- Use environment variables for secrets; never commit keys.
π NON-CORE (Good to know)
- REST vs GraphQL; pandas read_sql; connection pooling.
Worked example (source note: 9_Read_data_from_api_and_SQL_(1) (1).ipynb). Run it as a .py file. A notebook is optional.
Worked example. Lines that start with ! or % are notebook-only and are not valid in a .py file.
# --- Code cell 1 ---
# agenda
# read the data from api and convert that data into a dataframe
# creating a database and reading the data from sql and convert that data into a data frame
# --- Code cell 2 ---
import requests
import json
# --- Code cell 3 ---
import pandas as pd
# --- Code cell 4 ---
from pandas import json_normalize
# --- Code cell 5 ---
response = requests.get("https://api.mfapi.in/mf/114564", allow_redirects = True)
print(response.headers.get('content-type'))
content = response.content
# --- Code cell 6 ---
type(response)
# --- Code cell 7 ---
print(content)
# --- Code cell 8 ---
data = json.loads(content) ## parse a JSON string and convert it into a Python Dictionary.
# --- Code cell 9 ---
data
# --- Code cell 10 ---
type(data)
# --- Code cell 12 ---
data = json_normalize(data)# convert dictionary to dataframe
# --- Code cell 13 ---
data
# --- Code cell 14 ---
type(data)
# --- Code cell 15 ---
import pandas as pd
dataframe = pd.DataFrame.from_dict(data['data'][0])
#dataframe1 = pd.DataFrame.from_dict(data['data'][1])
# --- Code cell 16 ---
json_data=response.json()
meta=json_data["meta"]
meta
# --- Code cell 17 ---
dataframe
# --- Code cell 18 ---
dataframe
# --- Code cell 19 ---
dataframe.iloc[12:25,:1]
# --- Code cell 20 ---
status = data['status']
print(status)
# --- Code cell 22 ---
# this is a comment
# --- Code cell 23 ---
""" this is a
triple quotation
statment"""
# --- Code cell 24 ---
#SQLite is a C library that provides a lightweight
#disk-based database that doesnβt require a separate server process
# and allows accessing the database using SQL
# --- Code cell 25 ---
import sqlite3
# --- Code cell 26 ---
----Doc string/Multi line comment
# what ever you have written in """ is called as sql query upon importing sqlite3
# --- Code cell 27 ---
import pandas as pd
# --- Code cell 28 ---
conn = sqlite3.connect('machine_learning_database_new.db')
# --- Code cell 29 ---
c = conn.cursor() # this method used to read the sql query against the database
# c.execute() This method executes a SQL query against the database.
# --- Code cell 30 ---
c.execute('''
CREATE TABLE IF NOT EXISTS products
([product_id] INTEGER,[product_price] INTEGER, [product_name] TEXT)
''')
conn.commit()
# --- Code cell 31 ---
c.execute('''
INSERT INTO products (product_id, product_price, product_name)
VALUES
(1,43000,'Computer'),
(2,15000,'Printer'),
(3,7000,'Tablet'),
(4,12000,'Mobile'),
(5,5000,'Chair')
''')
conn.commit()
# --- Code cell 32 ---
c.execute('''
SELECT * FROM products
''')
#c.fetchall()The method fetches all (or all remaining) rows of a query result set
df = pd.DataFrame(c.fetchall(), columns=['product_id', 'product_price', 'product_name'])
# --- Code cell 33 ---
df
# --- Code cell 35 ---
#pyodbc is widely used in companies
#import pyodbc
#connection_string = ("Driver={ODBC Driver 17 for SQL Server};"
# "Server=USXXX00345,67800;"
# "Database=DB02;"
# "Trusted_Connection=yes;") # Based on server configurations user name and password maybe required
#connection = pyodbc.connect(connection_string)
# --- Code cell 36 ---
#data = pd.read_sql_query("SELECT TOP(100) * FROM associates", connection)
# --- Code cell 37 ---
#help(pd.read_sql_query)
Knowledge check
What should a requests call do before you trust response.json()?