Updating samarth branch from main

.dvc/.gitignore

@@ -1,3 +1,3 @@
+/cache
 /config.local
 /tmp
-/cache

.dvc/config

@@ -3,3 +3,5 @@
     autostage = true
 ['remote "myremote"']
     url = gs://stock_price_prediction_dataset/DVC
+['remote "gcs_remote"']
+    url = gs://stock_price_prediction_dataset/Data

.github/workflows/PyTest.yaml

@@ -0,0 +1,74 @@
+name: Python Test Workflow
+
+on:
+  push:
+    branches:
+      - main
+      - citest
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Step 1: Checkout the repository
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      # Step 2: Set up Python environment
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+
+      # Step 3: Install dependencies
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r pipeline/requirements.txt
+
+      # Step 4: Run tests and generate coverage report
+      - name: Run tests 
+        run: |
+          pip install coverage
+          coverage run -m pytest pipeline/airflow/tests --maxfail=1 --disable-warnings
+          coverage html
+
+      # Step 5: Upload coverage report as artifact
+      - name: Upload coverage report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report
+          path: htmlcov/
+
+      # Step 6: Install Google Cloud CLI
+      - name: Install Google Cloud CLI
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y google-cloud-cli
+
+      # Step 7: Decode GCP Service Account Key
+      - name: Decode and Write GCP Service Account Key
+        run: |
+          echo "${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}" | base64 -d > /tmp/gcp-key.json
+        shell: bash
+
+      # Step 8: Authenticate with GCP using the Temporary File
+      - name: Authenticate with GCP
+        env:
+          GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp-key.json
+        run: gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
+
+      # Step 9: Set GCP Project ID
+      - name: Set GCP Project ID
+        run: gcloud config set project ${{ secrets.GCP_PROJECT_ID }}
+
+      # Step 10: Upload coverage report to GCP bucket
+      - name: Upload coverage report to GCP bucket
+        env:
+          GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}
+        run: |
+           CURRENT_DATE=$(date +"%Y-%m-%d")
+           zip -r coverage-report.zip htmlcov/
+           gsutil -m cp coverage-report.zip gs://${GCS_BUCKET_NAME}/Pytest-reports/${CURRENT_DATE}/coverage-report.zip

.github/workflows/airflowtrigeer.yaml

@@ -0,0 +1,93 @@
+name: Trigger Airflow DAG Workflow
+
+on:
+  push:
+    branches:
+      - main
+      - citest
+  workflow_dispatch:
+
+jobs:
+  trigger_airflow_dag:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+      - name: Set up Docker Compose
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y docker-compose
+      - name: Initialize Airflow
+        working-directory: ./pipeline/airflow
+        run: |
+          docker-compose up airflow-init
+      - name: Start Airflow Services
+        working-directory: ./pipeline/airflow
+        run: |
+          docker-compose up -d
+      # # Step 6: Install Python packages inside Docker containers
+      # - name: Install Python Packages
+      #   working-directory: ./pipeline/airflow
+      #   run: |
+      #     docker-compose exec -T airflow-scheduler python3 -m pip install -r /opt/airflow/dags/requirements.txt
+      #     # docker-compose exec -T airflow-webserver python3 -m pip install -r /opt/airflow/dags/requirements.txt
+
+
+      # Step 6: Set permissions for Airflow logs inside the container
+      - name: Set permissions for Airflow logs
+        working-directory: ./pipeline/airflow
+        run: |
+          docker-compose exec -T --user root airflow-scheduler bash -c "chmod -R 777 /opt/airflow/logs/"
+      - name: Wait for Airflow to Initialize
+        working-directory: ./pipeline/airflow
+        run: |
+          timeout 300 bash -c 'until docker-compose exec -T airflow-webserver curl -f http://localhost:8080/health; do sleep 10; done'
+
+      # Step 9: Delete .pyc Files
+      - name: Delete .pyc Files
+        working-directory: ./pipeline/airflow
+        run: |
+          docker-compose exec -T airflow-scheduler find /opt/airflow -name \*.pyc -delete
+          docker-compose exec -T airflow-webserver find /opt/airflow -name \*.pyc -delete
+      - name: List DAG Import Errors
+        working-directory: ./pipeline/airflow
+        run: |
+          docker-compose exec -T airflow-scheduler airflow dags list-import-errors
+      - name: Show Airflow DAGs
+        working-directory: ./pipeline/airflow
+        run: |
+          docker-compose exec -T airflow-scheduler airflow dags list
+      - name: Trigger Airflow DAG
+        working-directory: ./pipeline/airflow
+        run: |
+          docker-compose exec -T airflow-scheduler airflow dags trigger -r manual_$(date +%Y%m%d%H%M%S) Group10_DataPipeline_MLOps
+      - name: Monitor DAG Execution
+        working-directory: ./pipeline/airflow
+        run: |
+          for i in {1..10}; do
+            STATUS=$(docker-compose exec -T airflow-scheduler airflow dags state Group10_DataPipeline_MLOps $(date +%Y-%m-%d))
+            echo "Current DAG status: $STATUS"
+            if [ "$STATUS" = "success" ]; then
+              echo "DAG completed successfully"
+              break
+            elif [ "$STATUS" = "failed" ]; then
+               echo "DAG failed"
+              exit 1
+            fi
+            sleep 60
+          done
+      - name: Stop Airflow Services
+        if: always()
+        working-directory: ./pipeline/airflow
+        run: docker-compose down --volumes --rmi all

.github/workflows/cloudbuild.yaml

@@ -0,0 +1,78 @@
+# steps:
+#   # Step 1: Clone the repository
+#   - name: 'gcr.io/cloud-builders/git'
+#     entrypoint: 'bash'
+#     args:
+#       - '-c'
+#       - |
+#         git clone https://github.com/IE7374-MachineLearningOperations/StockPricePrediction.git &&
+#         cd StockPricePrediction &&
+#         echo "Repository cloned successfully"
+
+#   # Step 2: Upload specific files to GCP Bucket
+#   - name: 'gcr.io/cloud-builders/gsutil'
+#     args:
+#       - '-m'
+#       - 'cp'
+#       - '-r'
+#       - 'StockPricePrediction/*.py'
+#       - 'StockPricePrediction/*.ipynb'
+#       - 'StockPricePrediction/*.pkl'
+#       - 'gs://stock_price_prediction_dataset/'
+
+#   # Step 3: Install dependencies
+#   - name: 'gcr.io/cloud-builders/pip'
+#     args:
+#       - 'install'
+#       - '-r'
+#       - 'StockPricePrediction/requirements.txt'
+
+# #   # Step 4: Train the model
+# #   - name: 'gcr.io/cloud-builders/python'
+# #     args:
+# #       - 'StockPricePrediction/train.py'
+
+# #   # Step 5: Validate the model
+# #   - name: 'gcr.io/cloud-builders/python'
+# #     args:
+# #       - 'StockPricePrediction/validate.py'
+
+# #   # Step 6: Conditional deployment if validation is successful
+# #   - name: 'gcr.io/cloud-builders/bash'
+# #     id: 'Check Validation'
+# #     args:
+# #       - '-c'
+# #       - |
+# #         ACCURACY=$(python StockPricePrediction/validate.py --get_accuracy) &&
+# #         if (( $(echo "$ACCURACY > 0.70" | bc -l) )); then
+# #           echo "Model accuracy is sufficient, proceeding with deployment";
+# #         else
+# #           echo "Model accuracy is insufficient, stopping deployment";
+# #           exit 1;
+# #         fi
+
+# #   # Step 7: Save the trained model to GCP Bucket
+# #   - name: 'gcr.io/cloud-builders/gsutil'
+# #     args:
+# #       - 'cp'
+# #       - 'StockPricePrediction/models/*.h5'
+# #       - 'gs://stock_price_prediction_dataset/trained_models/'
+
+# #   # Step 8: Run Unit Tests
+# #   - name: 'gcr.io/cloud-builders/python'
+# #     args:
+# #       - '-m'
+# #       - 'unittest'
+# #       - 'discover'
+# #       - '-s'
+# #       - 'StockPricePrediction/tests'
+
+# # artifacts:
+# #   objects:
+# #     location: 'gs://stock_price_prediction_dataset/artifacts/'
+# #     paths:
+# #       - 'StockPricePrediction/*.py'
+# #       - 'StockPricePrediction/*.ipynb'
+# #       - 'StockPricePrediction/*.h5'
+
+# # timeout: '1200s'

.github/workflows/model.yml

@@ -0,0 +1,85 @@
+name: Model Training
+
+on:
+  push:
+    branches:
+      - main
+      - citest  
+
+jobs:
+  train_model:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Step 1: Checkout repository
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      # Step 2: Set up Python
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.10.5'
+
+      # Step 3: Install all dependencies
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r pipeline/requirements.txt
+
+      # Step 4: Run Py Tests
+      - name: Run Tests
+        run: |
+          pytest pipeline/airflow/tests --maxfail=1 --disable-warnings
+
+      # Step 5: Authenticate with GCP
+      - name: Authenticate to GCP
+        env:
+          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+          GCP_REGION: ${{ secrets.GOOGLE_CLOUD_REGION }}
+          GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+        run: |
+          echo "${GCP_SERVICE_ACCOUNT_KEY}" | base64 --decode > ${HOME}/gcp-key.json
+          gcloud auth activate-service-account --key-file=${HOME}/gcp-key.json
+          gcloud config set project ${GCP_PROJECT_ID}
+          gcloud config set compute/region ${GCP_REGION}
+
+      # # Step 6: Conditional Model Training
+      # - name: Trigger Model Training
+      #   env:
+      #     GCP_BUCKET_NAME: ${{ secrets.GCP_BUCKET_NAME }}
+      #   run: |
+      #     if [ "${{ github.ref }}" == "refs/heads/main" ]; then
+      #       echo "Running production model training on GCP"
+      #       gcloud ai-platform jobs submit training model_training_$(date +%Y%m%d_%H%M%S) \
+      #         --region ${GCP_REGION} \
+      #         --module-name trainer.task \
+      #         --package-path ./trainer \
+      #         --python-version 3.10 \
+      #         --runtime-version 2.5 \
+      #         --job-dir gs://${GCP_BUCKET_NAME}/models/training_$(date +%Y%m%d_%H%M%S) \
+      #         -- \
+      #         --additional_training_args
+      #     else
+      #       echo "Running model training locally for testing"
+      #       python trainer/task.py --test_data ./data/test_data.csv
+      #     fi
+
+      # # Step 7: Save Training Logs as Artifacts
+      # - name: Upload Training Logs
+      #   if: always()
+      #   uses: actions/upload-artifact@v3
+      #   with:
+      #     name: training-logs
+      #     path: logs/
+
+      # # Step 8: Send notification on failure
+      # - name: Notify on failure
+      #   if: failure()
+      #   uses: actions/github-script@v6
+      #   with:
+      #     script: |
+      #       github.issues.createComment({
+      #         issue_number: context.issue.number,
+      #         body: 'Model training failed on `${{ github.ref }}` branch. Please check logs for details.'
+      #       })