Dataset.jl and Corresponding Tests

Project.toml

@@ -1,7 +1,16 @@
 name = "RidgeRegression"
 uuid = "739161c8-60e1-4c49-8f89-ff30998444b1"
-authors = ["Vivak Patel <vp314@users.noreply.github.com>"]
 version = "0.1.0"
+authors = ["Eton Tackett <etont@icloud.com>", "Vivak Patel <vp314@users.noreply.github.com>"]
+
+[deps]
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+RidgeRegression = "739161c8-60e1-4c49-8f89-ff30998444b1"
 
 [compat]
+CSV = "0.10.15"
+DataFrames = "1.8.1"
+Downloads = "1.7.0"
 julia = "1.12.4"

docs/make.jl

@@ -14,6 +14,7 @@ makedocs(;
     ),
     pages=[
         "Home" => "index.md",
+        "Design" => "design.md",
     ],
 )
 

src/RidgeRegression.jl

@@ -1,5 +1,11 @@
 module RidgeRegression
 
-# Write your package code here.
+using CSV
+using DataFrames
+using Downloads
+
+include("dataset.jl")
+
+export Dataset, csv_dataset, one_hot_encode
 
 end

src/dataset.jl

@@ -0,0 +1,119 @@
+"""
+    Dataset(name, X, y)
+
+Contains datasets for ridge regression experiments.
+
+# Fields
+- `name::String`: Name of dataset
+- `X::Matrix{Float64}`: Matrix of variables/features
+- `y::Vector{Float64}`: Target vector
+
+# Throws
+- `ArgumentError`: If rows in `X` does not equal length of `y`.
+
+!!! note
+    Used as the experimental unit for ridge regression experiments.
+"""
+struct Dataset
+    name::String
+    X::Matrix{Float64}
+    y::Vector{Float64}
+
+    function Dataset(name::String, X::AbstractMatrix, y::AbstractVector)
+        size(X, 1) == length(y) ||
+            throw(ArgumentError("X and y must have same number of rows"))
+
+        new(name, Matrix{Float64}(X), Vector{Float64}(y))
+    end
+end
+
+"""
+    one_hot_encode(Xdf::DataFrame; drop_first=true)
+
+One-hot encode categorical (string-like) features in `Xdf`.
+
+# Arguments
+- `Xdf::DataFrame`: Input DataFrame containing features and response vector `y`.
+
+# Keyword Arguments
+- `drop_first::Bool=true`: If `true`, drop the first dummy column for
+  each categorical feature to avoid multicollinearity.
+
+# Returns
+- `Matrix{Float64}`: A numeric matrix containing the encoded feature.
+"""
+function one_hot_encode(Xdf::DataFrame; drop_first::Bool = true)::Matrix{Float64}
+    n = nrow(Xdf)
+    cols = Vector{Vector{Float64}}()
+
+    for name in names(Xdf) #Selecting columns that aren't the target variable and pushing them to the columns.
+        col = Xdf[!, name]
+        if eltype(col) <: Real
+            push!(cols, Float64.(col))
+            continue
+        end
+
+        scol = string.(col) # Convert to string for categorical processing.
+        lv = unique(scol) #Get unique category levels.
+        ind = scol .== permutedims(lv) #Create indicator matrix for each level of the categorical variable.
+        #Permutedims is used to align the dimensions for broadcasting.
+        #Broadcasting compares each element of `scol` with each level in `lv`, resulting in a matrix where each column corresponds to a level and contains `true` for rows that match that level and `false` otherwise.
+
+        if drop_first && size(ind, 2) > 1 #Drop the first column of the indicator matrix to avoid multicollinearity if drop_first is true and there are multiple levels.
+            ind = ind[:, 2:end]
+        end
+
+        for j in 1:size(ind, 2)
+            push!(cols, Float64.(ind[:, j])) #Convert the boolean indicator columns to Float64 and add them to the list of columns.
+        end
+    end
+
+    p = length(cols)
+    X = Matrix{Float64}(undef, n, p)
+    for j in 1:p
+        X[:, j] = cols[j]
+    end
+
+    return Matrix{Float64}(X)
+
+end
+"""
+    csv_dataset(path_or_url; target_col, name="csv_dataset")
+
+Load a dataset from a CSV file or URL.
+
+# Arguments
+- `path_or_url::String`: Local file path or web URL containing CSV data.
+
+# Keyword Arguments
+- `target_col`: Column index or column name containing the response variable.
+- `name::String="csv_dataset"`: Dataset name.
+
+# Returns
+- `Dataset`: A dataset containing the encoded feature matrix `X`, response vector `y`, and dataset name.
+"""
+function csv_dataset(path_or_url::String;
+    target_col,
+    name::String = "csv_dataset"
+)
+
+    filepath =
+        startswith(path_or_url, "http") ?
+        Downloads.download(path_or_url) :
+        path_or_url
+
+    df = DataFrame(CSV.File(filepath)) #Read CSV file into a DataFrame.
+    df = dropmissing(df) #Remove rows with missing values.
+    Xdf = select(df, DataFrames.Not(target_col)) #Select all columns except the target column for features.
+
+    y = target_col isa Int ?
+        df[:, target_col] : #If target_col is an integer, use it as a column index to extract the target variable from the DataFrame.
+        df[:, Symbol(target_col)] #Extract the target variable based on whether target_col is an index or a name.
+
+
+    X = one_hot_encode(Xdf; drop_first = true)
+
+
+
+    return Dataset(name, Matrix{Float64}(X), Vector{Float64}(y))
+end

test/Project.toml

@@ -1,2 +1,9 @@
 [deps]
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+RidgeRegression = "739161c8-60e1-4c49-8f89-ff30998444b1"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[compat]
+CSV = "0.10"
+DataFrames = "1"

test/dataset_tests.jl

@@ -0,0 +1,65 @@
+using Test
+using DataFrames
+using CSV
+using RidgeRegression
+@testset "Dataset" begin
+    X = [1 2; 3 4]
+    y = [10, 20]
+    d = Dataset("toy", X, y)
+
+    @test d.name == "toy"
+    @test d.X isa Matrix{Float64}
+    @test d.y isa Vector{Float64}
+    @test size(d.X) == (2, 2)
+    @test length(d.y) == 2
+    @test d.X[1, 1] == 1.0
+    @test d.y[2] == 20.0
+
+    @test_throws ArgumentError Dataset("bad", X, [1, 2, 3])
+end
+
+@testset "one_hot_encode" begin
+    df = DataFrame(
+        A = ["red", "blue", "red", "green"],
+        B = [1, 2, 3, 4],
+        C = ["small", "large", "medium", "small"]
+    )
+
+    X = redirect_stdout(devnull) do
+        one_hot_encode(df; drop_first = true)
+    end
+
+    @test size(X) == (4, 5)
+    @test X[:, 3] == [1.0, 2.0, 3.0, 4.0]
+    @test all(x -> x == 0.0 || x == 1.0, X[:, [1,2,4,5]])
+    @test all(vec(sum(X[:, 1:2]; dims=2)) .<= 1)
+    @test all(vec(sum(X[:, 4:5]; dims=2)) .<= 1)
+end
+
+@testset "csv_dataset" begin
+    tmp = tempname() * ".csv"
+    df = DataFrame(
+        a = [1.0, 2.0, missing, 4.0],
+        b = ["x", "y", "y", "x"],
+        y = [10.0, 20.0, 30.0, 40.0]
+    )
+    CSV.write(tmp, df)
+
+    d = redirect_stdout(devnull) do
+        csv_dataset(tmp; target_col=:y, name="tmp")
+    end
+
+    @test d.name == "tmp"
+    @test d.X isa Matrix{Float64}
+    @test d.y isa Vector{Float64}
+
+    @test length(d.y) == 3
+    @test size(d.X, 1) == 3
+    @test d.y == [10.0, 20.0, 40.0]
+
+    d2 = redirect_stdout(devnull) do
+        csv_dataset(tmp; target_col=3, name="tmp2")
+    end
+    @test d2.y == [10.0, 20.0, 40.0]
+    @test size(d2.X, 1) == 3
+end

test/runtests.jl

@@ -2,5 +2,5 @@ using RidgeRegression
 using Test
 
 @testset "RidgeRegression.jl" begin
-    # Write your tests here.
+    include("dataset_tests.jl")
 end