Pry run and reload

.circleci/config.yml

@@ -1,33 +1,96 @@
-# Ruby CircleCI 2.0 configuration file
-#
-# Check https://circleci.com/docs/2.0/language-ruby/ for more details
-#
+# CircleCI 2.0 configuration file
 version: 2
-jobs:
-  build:
+x-defaults:
+  images:
     # TODO: bundler env variables --jobs=4 --retry=3
+    ruby: &image-ruby
+      image: "circleci/ruby:2.6.3"
+      environment:
+        BUNDLE_JOBS: "4"
+        BUNDLE_RETRY: "10"
+        BUNDLE_PATH: ../vendor/bundle
+        DEPLOY_ENV: "development"
+        RACK_ENV: "development"
+        POSTGRES_USER: "postgres"
+        POSTGRES_PASSWORD: ""
+        POSTGRES_HOST: "postgres"
+        POSTGRES_URI: "postgresql:///resources"
+  steps:
+    restore_cache: &step-restore_cache
+      restore_cache:
+        keys:
+          - v1-dependencies-{{ .Environment.CIRCLE_WORKFLOW_ID }}-{{ checksum "base/Gemfile.lock" }}
+    link_all: &step-link_all
+      run:
+        name: "link all services to the defaults"
+        command: "bin/local-link-all"
+workflows:
+  version: 2
+  primary:
+    jobs:
+      - setup
+      - lint:
+          requires:
+            - setup
+      - typecheck:
+          requires:
+            - setup
+      - build:
+          requires:
+            - setup
+jobs:
+  setup:
     docker:
-      - image: circleci/ruby:2.6.3
-        environment:
-          DEPLOY_ENV: "development"
-          RACK_ENV: "development"
-          POSTGRES_USER: "postgres"
-          POSTGRES_PASSWORD: ""
-          POSTGRES_HOST: "postgres"
-          POSTGRES_URI: "postgresql:///resources"
-      - image: circleci/postgres:11.2
+      - *image-ruby
+      - image: "circleci/postgres:11.2"
     steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "./_base/Gemfile.lock" }}
+      - "checkout"
+      - *step-restore_cache
+      - *step-link_all
+      - run:
+          name: "ensuring basic dependencies installed"
+          command: "sudo apt install postgresql-client"
       - run:
-          name: setup services
-          command: bin/local-setup
+          name: "wait for the database to be ready"
+          command: dockerize -wait tcp://localhost:5432 -timeout 1m
+      - run:
+          name: "setup the basics of the database"
+          command: "bin/local base ../.docker-compose/postgres/docker-entrypoint-initdb.d/create-users-and-groups.sh"
+      - run:
+          name: "install all libraries"
+          command: "bin/local base bundle install"
+      - run:
+          name: "run migrations"
+          command: "bin/local base bundle exec rake db:migrate"
       - save_cache:
-          paths:
-            - ./vendor/bundle
-          key: v1-dependencies-{{ checksum "./_base/Gemfile.lock" }}
+          paths: ["./vendor/bundle"]
+          key: v1-dependencies-{{ .Environment.CIRCLE_WORKFLOW_ID }}-{{ checksum "base/Gemfile.lock" }}
+  lint:
+    docker:
+      - *image-ruby
+    steps:
+      - "checkout"
+      - *step-restore_cache
+      - *step-link_all
+      - run:
+          name: "check against linter"
+          command: "bin/local-each bin/local-lint"
+  typecheck:
+    docker:
+      - *image-ruby
+    steps:
+      - "checkout"
+      - *step-restore_cache
+      - *step-link_all
+      - run:
+          name: "check against typechecker"
+          command: "bin/local-each bin/local-typecheck"
+  build:
+    machine: true
+    steps:
+      - "checkout"
+      - *step-restore_cache
+      - *step-link_all
       - run:
-          name: run tooling
-          command: "bin/local-check"
+          name: "build images"
+          command: "bin/local-each bin/docker-build-ci"

.docker-compose/postgres/docker-entrypoint-initdb.d/create-users-and-groups.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --host "$POSTGRES_HOST" --username "$POSTGRES_USERNAME" --dbname "template1" <<-EOSQL
+  BEGIN;
+    CREATE EXTENSION IF NOT EXISTS "citext";
+    CREATE EXTENSION IF NOT EXISTS "pgcrypto";
+    CREATE EXTENSION IF NOT EXISTS "cube";
+    CREATE EXTENSION IF NOT EXISTS "btree_gin";
+    CREATE EXTENSION IF NOT EXISTS "btree_gist";
+    CREATE EXTENSION IF NOT EXISTS "hstore";
+    CREATE EXTENSION IF NOT EXISTS "isn";
+    CREATE EXTENSION IF NOT EXISTS "ltree";
+    CREATE EXTENSION IF NOT EXISTS "lo";
+    CREATE EXTENSION IF NOT EXISTS "fuzzystrmatch";
+    CREATE EXTENSION IF NOT EXISTS "pg_buffercache";
+    CREATE EXTENSION IF NOT EXISTS "pgrowlocks";
+    CREATE EXTENSION IF NOT EXISTS "pg_prewarm";
+    CREATE EXTENSION IF NOT EXISTS "pg_stat_statements";
+    CREATE EXTENSION IF NOT EXISTS "pg_trgm";
+    CREATE EXTENSION IF NOT EXISTS "tablefunc";
+  COMMIT;
+EOSQL
+psql -v --username "$POSTGRES_USERNAME" --host "$POSTGRES_HOST" --dbname "template1" <<-EOSQL
+  DROP DATABASE IF EXISTS "resources";
+  CREATE DATABASE "resources";
+  GRANT ALL PRIVILEGES ON DATABASE "resources" TO "postgres";
+EOSQL

.dockerignore

@@ -1,10 +1,11 @@
 .bundle/
 .circleci/
+.DS_Store
+.gcloud/
 .git/
-.rubocop/
-services/
-gcloud/
+.gitpod/
+.docker-compose/
+.template/
 docs/
 tmp/
-.DS_Store
 .env

_gcloud/cloudbuild.yaml → .gcloud/cloudbuild.yaml

@@ -4,7 +4,7 @@ images:
   - "us.gcr.io/$PROJECT_ID/base:$SHORT_SHA"
   - "us.gcr.io/$PROJECT_ID/$_SERVICE_ID:latest"
   - "us.gcr.io/$PROJECT_ID/$_SERVICE_ID:$SHORT_SHA"
-timeout: "5m"
+timeout: "300s"
 steps:
   - name: "gcr.io/cloud-builders/docker"
     entrypoint: "sh"

.gitignore

@@ -10,7 +10,7 @@
 # Ignore all temporary files
 /tmp/*
 !/tmp/.keep
-/_services/*/data/
+/.docker-compose/*/data/
 
 # Ignore secret data
 /.env
@@ -32,4 +32,4 @@
 */.tool-versions
 
 # _base looks like linked information, but isn't
-!_base/*
+!base/*

.gitpod.yml

@@ -1,6 +1,6 @@
 # The Docker image to run your workspace in. Defaults to gitpod/workspace-full
-image: gitpod/workspace-postgres
+image: "krainboltgreene/workspace:latest"
 tasks:
-  - command: "bin/service-automatic-create && bin/local-setup"
+  - command: "bin/local base ../.docker-compose/postgres/docker-entrypoint-initdb.d/create-users-and-groups.sh && bin/service-automatic-create && bin/local-setup"
 ports:
   - port: 8080

.gitpod/Dockerfile

@@ -0,0 +1,17 @@
+FROM buildpack-deps:disco
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN yes | unminimize
+RUN apt-get update
+RUN apt-get install -yq bash-completion software-properties-common sudo
+RUN apt-get install -yq htop jq less multitail gnupg dirmngr postgresql postgresql-contrib postgresql-client ruby-full
+RUN apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/* /tmp/*
+### Gitpod user ###
+# '-l': see https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#user
+RUN useradd -l -u 33333 -G sudo -md /home/gitpod -s /bin/bash -p gitpod gitpod && sed -i.bkp -e 's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' /etc/sudoers
+ENV HOME /home/gitpod
+WORKDIR $HOME
+# custom Bash prompt
+RUN { echo && echo "PS1='\[\e]0;\u \w\a\]\[\033[01;32m\]\u\[\033[00m\] \[\033[01;34m\]\w\[\033[00m\] \\\$ '" ; } >> .bashrc
+# no root-owned files in the home directory
+RUN notOwnedFile=$(find . -not "(" -user gitpod -and -group gitpod ")" -print -quit) && { [ -z "$notOwnedFile" ] || { echo "Error: not all files/dirs in $HOME are owned by 'gitpod' user & group"; exit 1; } }

.local-env

@@ -1,6 +1,6 @@
 export POSTGRES_URI=postgresql:///resources
 export POSTGRES_HOST=localhost
-export POSTGRES_USERNAME=gitpod
+export POSTGRES_USERNAME=postgres
 export POSTGRES_PASSWORD=
 export DEPLOY_ENV=development
 export RACK_ENV=development

.tool-versions

@@ -1,4 +1,3 @@
 postgres 11.2
 ruby 2.6.3
 kubectl 1.14.1
-docker-slim 1.24

README.md

@@ -2,6 +2,10 @@
 
 A framework for serverless ruby on top of cloud run and also a collection of services for the farsight service.
 
+## hints
+
+  - Inside a pry session you can type `Pry.reload!` to reload all of your ruby code.
+
 
 ## tools
 
@@ -23,13 +27,18 @@ bin/local-pry mtgjson-import
 
 Creates a pry session with the entire context of the service loaded.
 
+*bin/local-run {{service}}*
+
+Excutes the main function of the service without a web server, using a mock response, request, and the regular database.
+
 
 ## services
 
 A service (or sometimes called function) is a singular semi-isolated piece of code that does one job. It has access to the request, the response, the router, a client list, and a database. Using these pieces the service will do some operation.
 
 For example, lets say we wanted a service that cleaned up old data. It would be called on a schedule (every 24h for example), it would pull all old records from the database, transform them to clean versions, and then store them in the database. Finally it would return status 200 (as a show that it completed successfully).
 
+
 ## how to setup
 
   0. Go to github.com/difference-engineers/farsight-functions
@@ -51,6 +60,13 @@ You'll know the process has worked because once gitpod has fully started your en
 Sometimes a service will need a specific library that most services wont need. For example, lets say you have a service that turns CSV documents into excel documents. You'll want to add the library like this:
 
 ```
-bin/local _base bundle add {{library}} --group={{service}}
-bin/local _base bundle add xlsx-reader --group=csv-to-xlsx
+bin/local base bundle add {{library}} --group={{service}}
+bin/local base bundle add xlsx-reader --group=mtgjson-import-cards
+```
+
+
+## How to think about services
+
+```
+(Data Gathering) -> (Decision Making) -> (Acting)
 ```

SERVICES

@@ -1,3 +1,3 @@
 mtgjson-import-cards
+mtgjson-import-keywords
 mtgjson-import-sets
-mtgjson-import-keywords

_base/Dockerfile → base/Dockerfile

@@ -8,6 +8,7 @@ ENV CONCURRENCY 2
 WORKDIR /home/root/application/
 
 COPY db/migrate/ db/migrate/
+COPY middleware/ middleware/
 COPY boot.rb .
 COPY configuration.rb .
 COPY application.rb .