task-lance-commit.yaml

# filename: task-lance-commit.yaml
apiVersion: tekton.dev/v1beta1
kind: Task
metadata:
  name: lance-commit
spec:
  params:
    - name: LANCEDB_STORAGE_TYPE
      type: string
    - name: LANCEDB_URI
      type: string
    - name: LANCEDB_DATA_URI
      type: string
    - name: LANCEDB_TABLE_NAME
      type: string
    - name: LANCEDB_FRAGMENTS_JSON_FOLDER
      type: string
    - name: LANCEDB_TABLE_SCHEMA_FOLDER
      type: string
  workspaces:
    - name: jobs
      description: A workspace to mount the job YAML.
  steps:
    - name: run-lance-commit-job
      image: quay.io/dataprep1/data-prep-kit/kubectl:latest
      env:
        - name: LANCEDB_STORAGE_TYPE
          value: $(params.LANCEDB_STORAGE_TYPE)
        - name: LANCEDB_URI
          value: $(params.LANCEDB_URI)
        - name: LANCEDB_DATA_URI
          value: $(params.LANCEDB_DATA_URI)
        - name: LANCEDB_TABLE_NAME
          value: $(params.LANCEDB_TABLE_NAME)
        - name: LANCEDB_FRAGMENTS_JSON_FOLDER
          value: $(params.LANCEDB_FRAGMENTS_JSON_FOLDER)
        - name: LANCEDB_TABLE_SCHEMA_FOLDER
          value: $(params.LANCEDB_TABLE_SCHEMA_FOLDER)
      script: |
        #!/bin/sh
        JOBFILE=/workspace/jobs/data-prep-kit/transforms/language/text_encoder/lance-commit-rayjob.yaml
        ##############################################################################
        ## Most jobs follow the same pattern, so we can templatize this in the future
        JOBNAME=`yq '.metadata.name' $JOBFILE`
        TMPFILE=$(mktemp)
        envsubst < "$JOBFILE" > "$TMPFILE"
        cat "$TMPFILE" && kubectl apply -f "$TMPFILE"
        echo "Watiing for Kubernetes rayjob/$JOBNAME to finish or timeout after 24h..."
        kubectl wait --for=jsonpath='{.status.jobDeploymentStatus}'=Complete rayjob/$JOBNAME --timeout=24h
        echo "Kubernetes rayjob/$JOBNAME completed. Check log for completion status..."
        ##############################################################################