Manohar

.github/workflows/PyTest.yaml

@@ -0,0 +1,84 @@
+name: Python Test Workflow
+
+on:
+  push:
+    branches:
+      - main
+      - citest
+
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+
+    steps:
+      # Step 1: Checkout the repository
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      # Step 2: Set up Python environment
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+
+      # Step 3: Install dependencies
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r pipeline/requirements.txt
+
+      # Step 4: Run tests and generate coverage report
+      - name: Run tests 
+        run: |
+          pip install coverage
+          coverage run -m pytest pipeline/airflow/tests --maxfail=2 --disable-warnings
+          coverage html
+          coverage xml
+
+      # Step 5: Upload coverage report as artifact
+      - name: Upload coverage report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report
+          path: htmlcov/
+
+      # Step 6: Upload XML coverage report as artifact
+      - name: Upload XML coverage report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report-xml
+          path: coverage.xml
+
+      # Step 7: Install Google Cloud CLI
+      - name: Install Google Cloud CLI
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y google-cloud-cli
+
+      # Step 8: Decode GCP Service Account Key
+      - name: Decode and Write GCP Service Account Key
+        run: |
+          echo "${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}" | base64 -d > /tmp/gcp-key.json
+        shell: bash
+
+      # Step 9: Authenticate with GCP using the Temporary File
+      - name: Authenticate with GCP
+        env:
+          GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp-key.json
+        run: gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
+
+      # Step 10: Set GCP Project ID
+      - name: Set GCP Project ID
+        run: gcloud config set project ${{ secrets.GCP_PROJECT_ID }}
+
+      # Step 11: Upload coverage reports to GCP bucket
+      - name: Upload coverage reports to GCP bucket
+        env:
+          GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }}
+        run: |
+           CURRENT_DATE=$(date +"%Y-%m-%d")
+           zip -r coverage-report.zip htmlcov/
+           gsutil -m cp coverage-report.zip gs://${GCS_BUCKET_NAME}/Pytest-reports/${CURRENT_DATE}/coverage-report.zip
+           gsutil cp coverage.xml gs://${GCS_BUCKET_NAME}/Pytest-reports/${CURRENT_DATE}/coverage.xml

.github/workflows/airflowtrigeer.yaml

@@ -9,58 +9,85 @@ on:
 
 jobs:
   trigger_airflow_dag:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
-      # Step 1: Checkout the repository
       - name: Checkout repository
         uses: actions/checkout@v2
 
-      # Step 2: Set up Python environment 
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
           python-version: '3.10'
 
-      # Step 3: Install Dependencies 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
-
-      # Step 4: Start Airflow using Docker Compose
+      - name: Set up Docker Compose
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y docker-compose
+      - name: Initialize Airflow
+        working-directory: ./pipeline/airflow
+        run: |
+          docker-compose up airflow-init
       - name: Start Airflow Services
+        working-directory: ./pipeline/airflow
         run: |
-          mkdir -p ./dags ./logs ./plugins
-          docker-compose -f pipeline/airflow/docker-compose.yaml up -d
+          docker-compose up -d
+      # # Step 6: Install Python packages inside Docker containers
+      # - name: Install Python Packages
+      #   working-directory: ./pipeline/airflow
+      #   run: |
+      #     docker-compose exec -T airflow-scheduler python3 -m pip install -r /opt/airflow/dags/requirements.txt
+      #     # docker-compose exec -T airflow-webserver python3 -m pip install -r /opt/airflow/dags/requirements.txt
 
-      # Step 5: Wait for Airflow Services to be Up
-      - name: Wait for Airflow to initialize
-        run: |
-          sleep 180  # Wait 3 minutes for Airflow to fully initialize
 
-      # Step 6: Trigger DAG via Airflow CLI
+      # Step 6: Set permissions for Airflow logs inside the container
+      - name: Set permissions for Airflow logs
+        working-directory: ./pipeline/airflow
+        run: |
+          docker-compose exec -T --user root airflow-scheduler bash -c "chmod -R 777 /opt/airflow/logs/"
+      - name: Wait for Airflow to Initialize
+        working-directory: ./pipeline/airflow
+        run: |
+          timeout 300 bash -c 'until docker-compose exec -T airflow-webserver curl -f http://localhost:8080/health; do sleep 10; done'
+
+      # Step 9: Delete .pyc Files
+      - name: Delete .pyc Files
+        working-directory: ./pipeline/airflow
+        run: |
+          docker-compose exec -T airflow-scheduler find /opt/airflow -name \*.pyc -delete
+          docker-compose exec -T airflow-webserver find /opt/airflow -name \*.pyc -delete
+      - name: List DAG Import Errors
+        working-directory: ./pipeline/airflow
+        run: |
+          docker-compose exec -T airflow-scheduler airflow dags list-import-errors
+      - name: Show Airflow DAGs
+        working-directory: ./pipeline/airflow
+        run: |
+          docker-compose exec -T airflow-scheduler airflow dags list
       - name: Trigger Airflow DAG
+        working-directory: ./pipeline/airflow
         run: |
-          docker-compose -f pipeline/airflow/docker-compose.yaml exec airflow-scheduler airflow dags trigger airflow  # Replace 'airflow' with your actual DAG ID
-
-      # Step 7: Monitor Airflow DAG Execution 
+          docker-compose exec -T airflow-scheduler airflow dags trigger -r manual_$(date +%Y%m%d%H%M%S) Group10_DataPipeline_MLOps
       - name: Monitor DAG Execution
+        working-directory: ./pipeline/airflow
         run: |
-          while true; do
-            STATUS=$(docker-compose -f pipeline/airflow/docker-compose.yaml exec airflow-scheduler airflow dags state airflow $(date +%Y-%m-%d))
+          for i in {1..10}; do
+            STATUS=$(docker-compose exec -T airflow-scheduler airflow dags state Group10_DataPipeline_MLOps $(date +%Y-%m-%d))
             echo "Current DAG status: $STATUS"
             if [ "$STATUS" = "success" ]; then
               echo "DAG completed successfully"
               break
             elif [ "$STATUS" = "failed" ]; then
-              echo "DAG failed"
+               echo "DAG failed"
               exit 1
             fi
-            sleep 60  # Wait before checking the status again
+            sleep 60
           done
-
-      # Step 8: Stop Airflow Services
       - name: Stop Airflow Services
         if: always()
-        run: docker-compose -f pipeline/airflow/docker-compose.yaml down --volumes --rmi all
+        working-directory: ./pipeline/airflow
+        run: docker-compose down --volumes --rmi all

.github/workflows/cloudbuild.yaml

@@ -1,78 +1,44 @@
+name: GCP CI/CD Workflow
+
 steps:
-  # Step 1: Clone the repository
-  - name: 'gcr.io/cloud-builders/git'
+  # Step 1: Run syncgcp.yaml
+  - name: 'gcr.io/cloud-builders/gcloud'
     entrypoint: 'bash'
     args:
       - '-c'
       - |
-        git clone https://github.com/IE7374-MachineLearningOperations/StockPricePrediction.git &&
-        cd StockPricePrediction &&
-        echo "Repository cloned successfully"
+        echo "Running syncgcp.yaml..."
+        gcloud builds submit --config=syncgcp.yaml
 
-  # Step 2: Upload specific files to GCP Bucket
-  - name: 'gcr.io/cloud-builders/gsutil'
+  # Step 2: Run pytest.yaml
+  - name: 'gcr.io/cloud-builders/gcloud'
+    entrypoint: 'bash'
     args:
-      - '-m'
-      - 'cp'
-      - '-r'
-      - 'StockPricePrediction/*.py'
-      - 'StockPricePrediction/*.ipynb'
-      - 'StockPricePrediction/*.pkl'
-      - 'gs://stock_price_prediction_dataset/'
+      - '-c'
+      - |
+        echo "Running pytest.yaml..."
+        gcloud builds submit --config=PyTest.yaml
 
-  # Step 3: Install dependencies
-  - name: 'gcr.io/cloud-builders/pip'
+  # Step 3: Run airflowtrigger.yaml
+  - name: 'gcr.io/cloud-builders/gcloud'
+    entrypoint: 'bash'
     args:
-      - 'install'
-      - '-r'
-      - 'StockPricePrediction/requirements.txt'
-
-#   # Step 4: Train the model
-#   - name: 'gcr.io/cloud-builders/python'
-#     args:
-#       - 'StockPricePrediction/train.py'
-
-#   # Step 5: Validate the model
-#   - name: 'gcr.io/cloud-builders/python'
-#     args:
-#       - 'StockPricePrediction/validate.py'
-
-#   # Step 6: Conditional deployment if validation is successful
-#   - name: 'gcr.io/cloud-builders/bash'
-#     id: 'Check Validation'
-#     args:
-#       - '-c'
-#       - |
-#         ACCURACY=$(python StockPricePrediction/validate.py --get_accuracy) &&
-#         if (( $(echo "$ACCURACY > 0.70" | bc -l) )); then
-#           echo "Model accuracy is sufficient, proceeding with deployment";
-#         else
-#           echo "Model accuracy is insufficient, stopping deployment";
-#           exit 1;
-#         fi
-
-#   # Step 7: Save the trained model to GCP Bucket
-#   - name: 'gcr.io/cloud-builders/gsutil'
-#     args:
-#       - 'cp'
-#       - 'StockPricePrediction/models/*.h5'
-#       - 'gs://stock_price_prediction_dataset/trained_models/'
-
-#   # Step 8: Run Unit Tests
-#   - name: 'gcr.io/cloud-builders/python'
-#     args:
-#       - '-m'
-#       - 'unittest'
-#       - 'discover'
-#       - '-s'
-#       - 'StockPricePrediction/tests'
-
-# artifacts:
-#   objects:
-#     location: 'gs://stock_price_prediction_dataset/artifacts/'
-#     paths:
-#       - 'StockPricePrediction/*.py'
-#       - 'StockPricePrediction/*.ipynb'
-#       - 'StockPricePrediction/*.h5'
+      - '-c'
+      - |
+        echo "Running airflowtrigger.yaml..."
+        gcloud builds submit --config=airflowtrigger.yaml
 
-# timeout: '1200s'
+  # Step 4: Run model.yaml
+  - name: 'gcr.io/cloud-builders/gcloud'
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        echo "Running model.yaml..."
+        gcloud builds submit --config=model.yaml
+
+# Trigger by GitHub Push
+trigger:
+  event:
+    type: PUSH
+    branch: 'main' 

.github/workflows/deploy.yaml

@@ -0,0 +1,83 @@
+name: Deploy and Monitor Vertex AI Model
+
+on:
+  push:
+    branches:
+      - main  
+      - citest
+
+jobs:
+  deploy-and-monitor:
+    runs-on: ubuntu-22.04
+
+    steps:
+    # Step 1: Checkout Code
+    - name: Checkout Code
+      uses: actions/checkout@v3
+
+ # Step 2: Set up Google Cloud SDK
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v0.2.0
+      with:
+        project_id: ${{ secrets.GCP_PROJECT_ID }}
+        service_account_key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+        export_default_credentials: true
+
+    # Step 3: Deploy in Vertex AI
+    - name: Deploy Model to Vertex AI
+      run: |
+          echo "Deploying model to Vertex AI..."
+          gcloud ai endpoints create \
+            --region=${{ secrets.GOOGLE_CLOUD_REGION }} \
+            --display-name=mlops-group10-deploy
+
+          ENDPOINT_ID=$(gcloud ai endpoints list \
+            --region=${{ secrets.GOOGLE_CLOUD_REGION }} \
+            --filter="display_name=mlops-group10-deploy" \
+            --format="value(name)")
+          echo "Endpoint ID: $ENDPOINT_ID"
+
+          echo "Deploying model to endpoint..."
+          gcloud ai endpoints deploy-model $ENDPOINT_ID \
+            --region=${{ secrets.GOOGLE_CLOUD_REGION }} \
+            --model=3792457496757534720 \
+            --display-name=mlops-group10-deploy \
+            --machine-type=n1-standard-4
+
+          echo "Model deployed successfully to endpoint $ENDPOINT_ID!"
+
+    # Step 4: Monitor Deployment Status
+    - name: Monitor Deployment Status
+      run: |
+        echo "Monitoring deployment status..."
+        DEPLOYMENT_STATUS=$(gcloud ai endpoints describe $ENDPOINT_ID --region=${{ secrets.GOOGLE_CLOUD_REGION }} --format="value(deployedModels.state)")
+        echo "Deployment status: $DEPLOYMENT_STATUS"
+        if [[ "$DEPLOYMENT_STATUS" != "DEPLOYED" ]]; then
+          echo "Deployment failed or is incomplete. Status: $DEPLOYMENT_STATUS"
+          exit 1
+        fi
+        echo "Deployment is successful. Status: $DEPLOYMENT_STATUS"
+
+    # Step 5: Fetch Logs for Deployment
+    - name: Fetch Deployment Logs
+      run: |
+        echo "Fetching logs for deployment..."
+        gcloud logging read "resource.labels.endpoint_id=$ENDPOINT_ID" \
+          --format="table(timestamp, textPayload)" \
+          --limit=50
+
+    # Step 6: Notify Deployment Status
+    - name: Notify Deployment Status
+      uses: actions/github-script@v5
+      with:
+        script: |
+          const deploymentStatus = `DEPLOYED`;
+          const message = deploymentStatus === "DEPLOYED"
+            ? " Deployment to Vertex AI completed successfully!"
+            : " Deployment to Vertex AI failed. Check logs for details.";
+          github.repos.createCommitComment({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            commit_sha: context.sha,
+            body: message,
+          });