Using TF-IDF with Logistic Regression for Sentiment Analysis

best_C = best["params"]["C"] best_solver = best["params"]["solver"] final_pipe = Pipeline([ ("scaler", StandardScaler()), ("clf", LogisticRegression( C=best_C, solver=best_solver, penalty="l2", max_iter=2000, random_state=42 )) ]) with mlflow.start_run(run_name="final_model_run") as final_run: final_pipe.fit(X_train, y_train) proba = final_pipe.predict_proba(X_test)[:, 1] pred = (proba >= 0.5).astype(int) metrics = { "test_auc": float(roc_auc_score(y_test, proba)), "test_accuracy": float(accuracy_score(y_test, pred)), "test_precision": float(precision_score(y_test, pred, zero_division=0)), "test_recall": float(recall_score(y_test, pred, zero_division=0)), "test_f1": float(f1_score(y_test, pred, zero_division=0)), } mlflow.log_metrics(metrics) mlflow.log_params({"C": best_C, "solver": best_solver, "model": "LogisticRegression+StandardScaler"}) input_example = X_test.iloc[:5].copy() signature = infer_signature(input_example, final_pipe.predict_proba(input_example)[:, 1]) model_info = mlflow.sklearn.log_model( sk_model=final_pipe, artifact_path="model", signature=signature, input_example=input_example, registered_model_name=None, ) print("Final run_id:", final_run.info.run_id) print("Logged model URI:", model_info.model_uri) eval_df = X_test.copy() eval_df["label"] = y_test.values eval_result = mlflow.models.evaluate( model=model_info.model_uri, data=eval_df, targets="label", model_type="classifier", evaluators="default", ) eval_summary = { "metrics": {k: float(v) if isinstance(v, (int, float, np.floating)) else str(v) for k, v in eval_result.metrics.items()}, "artifacts": {k: str(v) for k, v in eval_result.artifacts.items()}, } mlflow.log_dict(eval_summary, "evaluation/eval_summary.json")

!pip install -q pyspark==3.5.1 from pyspark.sql import SparkSession, functions as F, Window from pyspark.sql.types import IntegerType, StringType, StructType, StructField, FloatType from pyspark.ml.feature import StringIndexer, VectorAssembler from pyspark.ml.classification import LogisticRegression from pyspark.ml.evaluation import MulticlassClassificationEvaluator spark = (SparkSession.builder.appName("ColabSparkAdvancedTutorial") .master("local[*]") .config("spark.sql.shuffle.partitions", "4") .getOrCreate()) print("Spark version:", spark.version) data = [ (1, "Alice", "IN", "2025-10-01", 56000.0, "premium"), (2, "Bob", "US", "2025-10-03", 43000.0, "standard"), (3, "Carlos", "IN", "2025-09-27", 72000.0, "premium"), (4, "Diana", "UK", "2025-09-30", 39000.0, "standard"), (5, "Esha", "IN", "2025-10-02", 85000.0, "premium"), (6, "Farid", "AE", "2025-10-02", 31000.0, "basic"), (7, "Gita", "IN", "2025-09-29", 46000.0, "standard"), (8, "Hassan", "PK", "2025-10-01", 52000.0, "premium"), ] schema = StructType([ StructField("id", IntegerType(), False), StructField("name", StringType(), True), StructField("country", StringType(), True), StructField("signup_date", StringType(), True), StructField("income", FloatType(), True), StructField("plan", StringType(), True), ]) df = spark.createDataFrame(data, schema) df.show()

This tutorial provides an explanation of adjusted odds ratios, including a formal definition and several examples.

Gallery examples: Probability Calibration curves Plot classification probability Column Transformer with Mixed Types Pipelining: chaining a PCA and a logistic regression Feature transformations wit...

Gallery examples: Probability Calibration curves Plot classification probability Column Transformer with Mixed Types Pipelining: chaining a PCA and a logistic regression Feature transformations wit...

best_C = best["params"]["C"] best_solver = best["params"]["solver"] final_pipe = Pipeline([ ("scaler", StandardScaler()), ("clf", LogisticRegression( C=best_C, solver=best_solver, penalty="l2", max_iter=2000, random_state=42 )) ]) with mlflow.start_run(run_name="final_model_run") as final_run: final_pipe.fit(X_train, y_train) proba = final_pipe.predict_proba(X_test)[:, 1] pred = (proba >= 0.5).astype(int) metrics = { "test_auc": float(roc_auc_score(y_test, proba)), "test_accuracy": float(accuracy_score(y_test, pred)), "test_precision": float(precision_score(y_test, pred, zero_division=0)), "test_recall": float(recall_score(y_test, pred, zero_division=0)), "test_f1": float(f1_score(y_test, pred, zero_division=0)), } mlflow.log_metrics(metrics) mlflow.log_params({"C": best_C, "solver": best_solver, "model": "LogisticRegression+StandardScaler"}) input_example = X_test.iloc[:5].copy() signature = infer_signature(input_example, final_pipe.predict_proba(input_example)[:, 1]) model_info = mlflow.sklearn.log_model( sk_model=final_pipe, artifact_path="model", signature=signature, input_example=input_example, registered_model_name=None, ) print("Final run_id:", final_run.info.run_id) print("Logged model URI:", model_info.model_uri) eval_df = X_test.copy() eval_df["label"] = y_test.values eval_result = mlflow.models.evaluate( model=model_info.model_uri, data=eval_df, targets="label", model_type="classifier", evaluators="default", ) eval_summary = { "metrics": {k: float(v) if isinstance(v, (int, float, np.floating)) else str(v) for k, v in eval_result.metrics.items()}, "artifacts": {k: str(v) for k, v in eval_result.artifacts.items()}, } mlflow.log_dict(eval_summary, "evaluation/eval_summary.json")

!pip install -q pyspark==3.5.1 from pyspark.sql import SparkSession, functions as F, Window from pyspark.sql.types import IntegerType, StringType, StructType, StructField, FloatType from pyspark.ml.feature import StringIndexer, VectorAssembler from pyspark.ml.classification import LogisticRegression from pyspark.ml.evaluation import MulticlassClassificationEvaluator spark = (SparkSession.builder.appName("ColabSparkAdvancedTutorial") .master("local[*]") .config("spark.sql.shuffle.partitions", "4") .getOrCreate()) print("Spark version:", spark.version) data = [ (1, "Alice", "IN", "2025-10-01", 56000.0, "premium"), (2, "Bob", "US", "2025-10-03", 43000.0, "standard"), (3, "Carlos", "IN", "2025-09-27", 72000.0, "premium"), (4, "Diana", "UK", "2025-09-30", 39000.0, "standard"), (5, "Esha", "IN", "2025-10-02", 85000.0, "premium"), (6, "Farid", "AE", "2025-10-02", 31000.0, "basic"), (7, "Gita", "IN", "2025-09-29", 46000.0, "standard"), (8, "Hassan", "PK", "2025-10-01", 52000.0, "premium"), ] schema = StructType([ StructField("id", IntegerType(), False), StructField("name", StringType(), True), StructField("country", StringType(), True), StructField("signup_date", StringType(), True), StructField("income", FloatType(), True), StructField("plan", StringType(), True), ]) df = spark.createDataFrame(data, schema) df.show()

## Installed Key Python Libraries for Machine Learning Checking the version of the pip installed: PANDAS Importing pandas for the machine learning code SKICIT-LEARN Importing skicit learn MATPLOTLIB Importing matplotlib ALGORITHM from pandas import read_csv from matplotlib import pyplot from sklearn.linear_model import LogisticRegression from sklearn.model_selection import train_test_split import joblib # Step 1: Load dataset filename = "Iris.csv" data = read_csv(filename) # Step 2: Display data shape and preview print("Shape of the dataset:", data.shape) print("First 20 rows:\n", data.head(20)) # Step 3: Plot and save histograms silently data.hist() pyplot.savefig("histograms.png") pyplot.close() # Close the plot so it doesn't show up in prompt # Step 4: Plot and save density plots silently data.plot(kind='density', subplots=True, layout=(3,3), sharex=False) pyplot.savefig("density_plots.png") pyplot.close() # Step 5: Convert to NumPy array and extract features/labels array = data.values X = array[:, 1:5] # Features: Sepal/Petal measurements Y = array[:, 5] # Target: Species # Step 6: Split data into training (67%) and testing (33%) test_size = 0.33 seed = 7 X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=test_size, random_state=seed) # Step 7: Create and train logistic regression model model = LogisticRegression(max_iter=200) model.fit(X_train, Y_train) # Step 8: Evaluate and display accuracy result = model.score(X_test, Y_test) print("Accuracy: {:.2f}%".format(result * 100)) # Step 9: Save the trained model to a file joblib.dump(model, "logistic_model.pkl") IRIS DATASET OUTPUT DENSITY_PLOTS HISTOGRAM BRANCH CREATION IN GITHUB TO STORE THE CODE:

I have a imbalanced dataset and I want the the output as probabilities and not labels. Hence using Logistic Regression seemed to be the obvious choice. However the classsifer started predicting all

This guide demonstrates how to use scikit-learn’s callbacks on compatible estimators. For information about how to implement the callback API, you can refer to the following sections of the develop...

Binary logistic regression is one of the most widely used statistical tools. The method uses odds, log odds, and odds ratios, which are difficult to understand and interpret. Understanding of logisti....

Background: Medication compliance, which refers to the extent to which patients correctly adhere to prescribed regimens, is influenced by various psychological, behavioral, and demographic factors. When analyzing these factors, challenges such as…

A loosely rendered coastal vista seen from atop a grassy bluff: a slope of dark green vegetation and pale sand falls to a narrow beach at left, while a calm, grey-blue sea dotted with five small sailboats stretches under a broad, cloudy sky; a low headland fades into the distance on the horizon.