Initial OSS snapshot

.github/ISSUE_TEMPLATE/bug.yml

@@ -0,0 +1,60 @@
+# See https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-issue-forms
+# and https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-githubs-form-schema
+name: Bug Report
+description: Something is not working with the Lakeflow Framework
+title: "[BUG]: "
+labels: ["bug", "needs-triage"]
+# assignees:
+#   - dlt_framework-write
+body:
+  - type: checkboxes
+    attributes:
+      label: Is there an existing issue for this?
+      description: Please search to see if an issue already exists for the bug you encountered.
+      options:
+      - label: I have searched the existing issues
+        required: true
+  - type: textarea
+    attributes:
+      label: Current Behavior
+      description: |
+        A concise description of what you're experiencing.
+        **Do not paste links to attachments with logs and/or images, as all issues will attachments will get deleted.**
+        Use the `Relevant log output` field to paste redacted log output without personal identifying information (PII).
+        You can Ctrl/Cmd+V the screenshot, which would appear as a rendered image if it doesn't contain any PII.
+    validations:
+      required: false
+  - type: textarea
+    attributes:
+      label: Expected Behavior
+      description: A concise description of what you expected to happen.
+    validations:
+      required: false
+  - type: textarea
+    attributes:
+      label: Steps To Reproduce
+      description: Steps to reproduce the behavior.
+      placeholder: |
+        1. In this environment...
+        1. With this config...
+        1. Run '...'
+        1. See error...
+    validations:
+      required: false
+  - type: dropdown
+    id: channel
+    attributes:
+      label: Channel
+      description: What SDP Channel are you running on?
+      options:
+        - CURRENT
+        - PREVIEW
+      default: 0
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+      render: shell

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,9 @@
+blank_issues_enabled: false
+contact_links:
+  - name: General Databricks questions
+    url: https://help.databricks.com/
+    about: Issues related to Databricks and not related to Lakeflow Framework
+
+  - name: Lakeflow Framework Documentation
+    url: https://github.com/erik-seefeld_data/dlt_framework/tree/main/docs
+    about: Documentation about Lakeflow Framework

.github/ISSUE_TEMPLATE/feature.yml

@@ -0,0 +1,34 @@
+# See https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-issue-forms
+# and https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-githubs-form-schema
+name: Feature Request
+description: This new feauture / functiona;ity is required for the Lakeflow Framework
+title: "[FEATURE]: "
+labels: ["enhancement", "needs-triage"]
+# assignees:
+#   - dlt_framework-write
+body:
+  - type: checkboxes
+    attributes:
+      label: Is there an existing issue for this?
+      description: Please search to see if an issue already exists for the feature request you're willing to submit
+      options:
+      - label: I have searched the existing issues
+        required: true
+  - type: textarea
+    attributes:
+      label: Problem statement
+      description: A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Proposed Solution
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Additional Context
+      description: Add any other context, references or screenshots about the feature request here.
+    validations:
+      required: false

.github/workflows/develop-build.yml

@@ -0,0 +1,37 @@
+name: Dev Version on Merge to Develop
+
+on:
+  pull_request:
+    branches:
+      - develop
+    types:
+      - closed
+
+jobs:
+  dev-version:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Get latest tag
+        id: get_tag
+        run: |
+          TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0")
+          echo "Latest tag: $TAG"
+          echo "tag=$TAG" >> $GITHUB_OUTPUT
+
+      - name: Generate pre-release version
+        run: |
+          TAG=${{ steps.get_tag.outputs.tag }}
+          VERS=${TAG#v}
+          IFS='.' read -r MAJOR MINOR PATCH <<< "$VERS"
+          PATCH=$((PATCH + 1))
+          SHA=$(git rev-parse --short HEAD)
+          VERSION="${MAJOR}.${MINOR}.${PATCH}-dev.${SHA}"
+          echo $VERSION > VERSION
+          echo "Version: $VERSION"

.github/workflows/loc.yml

@@ -0,0 +1,44 @@
+name: LOC Badge
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  lines-of-code:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Count lines of code
+        run: |
+          curl -s https://raw.githubusercontent.com/AlDanial/cloc/master/cloc | perl - --csv --out=loc.csv .
+          LOC=$(tail -n1 loc.csv | cut -d',' -f5)
+          echo "Lines of code: $LOC"
+
+      # Update README with LOC badge
+      - name: Update README.md with LOC badge
+        run: |
+          # Define the badge
+          LOC_BADGE="![Lines of Code](https://img.shields.io/badge/lines_of_code-${LOC}-blue)"
+
+          # Check if README.md contains the badge
+          if grep -q "![Lines of Code]" README.md; then
+            # Update the existing badge
+            sed -i "s|!\[Lines of Code\](.*)|$LOC_BADGE|" README.md
+          else
+            # Add the badge at the end of README.md
+            echo -e "\n$LOC_BADGE" >> README.md
+          fi
+
+      # Commit and push changes
+      - name: Commit and push changes
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git add README.md
+          git commit -m "Update Lines of Code badge"
+          git push
+
+

.github/workflows/main-build.yml

@@ -0,0 +1,59 @@
+name: Version and Tag on Merge to Main
+
+on:
+  pull_request:
+    branches:
+      - main
+    types:
+      - closed
+
+jobs:
+  tag-version:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # So tags are available
+
+      - name: Get latest tag
+        id: get_tag
+        run: |
+          LAST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0")
+          echo "LAST_TAG=$LAST_TAG"
+          echo "last_tag=$LAST_TAG" >> $GITHUB_OUTPUT
+
+      - name: Compute next version
+        id: bump
+        run: |
+          OLD=${{ steps.get_tag.outputs.last_tag }}
+          VERS=${OLD#v}
+          IFS='.' read -r MAJOR MINOR PATCH <<< "$VERS"
+          PATCH=$((PATCH + 1))
+          NEW="v$MAJOR.$MINOR.$PATCH"
+          echo "NEW_VERSION=$NEW"
+          echo "new_version=$NEW" >> $GITHUB_OUTPUT
+
+      - name: Fail if version is not incremented
+        run: |
+          if [ "${{ steps.get_tag.outputs.last_tag }}" = "${{ steps.bump.outputs.new_version }}" ]; then
+            echo "❌ Version not incremented. Current tag (${{ steps.get_tag.outputs.last_tag }}) is the same as the new tag."
+            exit 1
+          fi
+
+      - name: Write VERSION file
+        run: |
+          echo "${{ steps.bump.outputs.new_version }}" | sed 's/^v//' > VERSION
+          cat VERSION
+
+      - name: Tag and push
+        env:
+          TAG: ${{ steps.bump.outputs.new_version }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git config user.name "github-actions"
+          git config user.email "github-actions@github.com"
+          git tag $TAG
+          git push origin $TAG

.github/workflows/main-docs.yml

@@ -0,0 +1,58 @@
+name: Deploy Lakeflow Framework Documentation to GitHub Pages
+
+on:
+  push:
+    branches: ["main"]
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on:
+      group: databricks-solutions-protected-runner-group
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-docs.txt
+
+      - name: Build HTML
+        run: |
+          cd docs
+          make html
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v4
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: 'docs/build/html'
+
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on:
+      group: databricks-solutions-protected-runner-group
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.gitignore

@@ -1,5 +1,10 @@
-*conf*.json
+.databricks/
+build/
+dist/
+__pycache__/
+*.egg-info
+.venv/
+scratch/**
+!scratch/README.md
 .DS_Store
-__pycache__
-.idea/
-.env
+**/.DS_Store