s3 workflow with gcp

.github/workflows/base-build-image-gcp.yml

@@ -0,0 +1,71 @@
+name: Build base images (Generally from basin repo)
+
+on:
+  workflow_call:
+    inputs:
+      image_tag:
+        required: true
+        type: string
+      fail_on_scan:
+        default: true
+        type: boolean
+
+jobs:
+  docker-build-and-push:
+
+    runs-on: ubuntu-22.04 #ubuntu-latest
+
+    steps:
+    - id: get-id
+      name: Get a unique tag for this build
+      run: |
+        echo "DOCKER_IMAGE=${{ vars.GCP_DOCKER_REGISTRY }}/${{ github.repository }}:${{ inputs.image_tag }}" >> "$GITHUB_OUTPUT";
+
+    - name: Print image name
+      run: |
+        echo "${{ steps.get-id.outputs.DOCKER_IMAGE }}";
+
+    - uses: actions/checkout@v4
+
+    # ✅ 1) Auth to GCP (this is where your SA key is used)
+    - name: Auth to GCP
+      uses: google-github-actions/auth@v2
+      with:
+        # using your existing secret that contains the SA JSON
+        credentials_json: ${{ secrets.GCP_SA_KEY }}
+
+      # ✅ 2) Install gcloud (no creds here)
+    - name: Set up gcloud
+      uses: google-github-actions/setup-gcloud@v2
+      with:
+        project_id: ${{ vars.GCP_PROJECT_ID }}
+        export_default_credentials: true
+
+    - name: Configure Docker for GAR
+      run: |
+        gcloud auth configure-docker ${{vars.GCP_REGION}}-docker.pkg.dev
+
+    - name: Build and push the Docker image
+      run: |
+        docker build \
+          --file context/Dockerfile \
+          --tag ${{ steps.get-id.outputs.DOCKER_IMAGE }} \
+          ./context;
+
+    - name: Container details
+      run: |
+        IMAGE_SIZE=`docker inspect -f "{{ .Size }}" ${{ steps.get-id.outputs.DOCKER_IMAGE }} | numfmt --to=si`;
+        echo "$IMAGE_SIZE container ${{ steps.get-id.outputs.DOCKER_IMAGE }}";
+
+#    - name: Scan Docker Image for vulnerabilities with Grype
+#      uses: anchore/scan-action@v6
+#      with:
+#        image: ${{ steps.get-id.outputs.DOCKER_IMAGE }}
+#        cache-db: true #Cache Grype DB in Github Actions
+#        output-format: table
+#        only-fixed: true
+#        severity-cutoff: critical
+#        fail-build: ${{ inputs.fail_on_scan }}
+
+    - name: Push the container image
+      run: docker push ${{ steps.get-id.outputs.DOCKER_IMAGE }}

.github/workflows/base-build-image.yml

@@ -6,6 +6,9 @@ on:
       image_tag:
         required: true
         type: string
+      fail_on_scan:
+        default: true
+        type: boolean
 
 jobs:
   docker-build-and-push:
@@ -52,7 +55,7 @@ jobs:
         output-format: table
         only-fixed: true
         severity-cutoff: critical
-        fail-build: true
+        fail-build: ${{ inputs.fail_on_scan }}
 
     - name: Push the container image
       run: docker push ${{ steps.get-id.outputs.DOCKER_IMAGE }}

.github/workflows/dispatch-container-base.yml

@@ -1,4 +1,4 @@
-name: Build base images (Generally from basin repo)
+name: Build base images from code repos
 
 on:
   workflow_call:

.github/workflows/push-code-test.yml

@@ -0,0 +1,30 @@
+name: Run Tests
+# Secrets can only viewed in "push" events. Not pull_request events.
+# That's why this step needs to be called on push, and not on pull_request (to read npm password).
+
+on:
+  workflow_call:
+
+jobs:
+
+  push-container-scan:
+    runs-on: ubuntu-22.04
+
+    steps:
+
+    - uses: actions/checkout@v4
+      
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 22
+        registry-url: ${{ vars.NPM_REGISTRY }}
+        token: ${{ secrets.NPM_TOKEN }}
+
+    - name: Install npm dependencies
+      run: |
+        npm install -g pnpm
+        pnpm install
+
+    - name: Run Tests
+      run: |
+        pnpm test

.github/workflows/push-container-gcp.yml

@@ -0,0 +1,63 @@
+name: Reusable container push workflow
+
+on:
+  workflow_call:
+
+env:
+  REPO: ${{ github.repository }}
+
+jobs:
+
+  push-container:
+    runs-on: ubuntu-22.04
+    steps:
+      - id: get-id
+        name: Get a unique tag for this build
+        run: |
+          SHA=${{ github.sha }}; BRANCH_NAME=${{ github.base_ref || github.ref_name }};
+          BUILD_ID=$BRANCH_NAME-${SHA:0:8};
+          DOCKER_IMAGE="${{vars.GCP_DOCKER_REGISTRY}}/$REPO:$BUILD_ID"
+          echo "BUILD_ID=$BUILD_ID" >> "$GITHUB_OUTPUT";
+          echo "DOCKER_IMAGE=$DOCKER_IMAGE" >> "$GITHUB_OUTPUT";
+
+      - name: Print build id and image name
+        run: |
+          echo "BUILD_ID: ${{ steps.get-id.outputs.BUILD_ID }}"; 
+          echo "DOCKER_IMAGE: ${{ steps.get-id.outputs.DOCKER_IMAGE }}";
+
+      - uses: actions/checkout@v4
+
+        # ✅ 1) Auth to GCP (this is where your SA key is used)
+      - name: Auth to GCP
+        uses: google-github-actions/auth@v2
+        with:
+          # using your existing secret that contains the SA JSON
+          credentials_json: ${{ secrets.GCP_SA_KEY }}
+
+        # ✅ 2) Install gcloud (no creds here)
+      - name: Set up gcloud
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          project_id: ${{ vars.GCP_PROJECT_ID }}
+          export_default_credentials: true
+
+      - name: Configure Docker for GAR
+        run: |
+          gcloud auth configure-docker ${{vars.GCP_REGION}}-docker.pkg.dev
+
+      - name: Build the container image
+        run: |
+          docker build \
+            --build-arg BUILD_STEP=container \
+            --build-arg PUBLIC_BUILD_VERSION=${{ steps.get-id.outputs.BUILD_ID }} \
+            --file fab/d/actions-build.Dockerfile \
+            --tag ${{ steps.get-id.outputs.DOCKER_IMAGE }} \
+            .;
+
+      - name: Container details
+        run: |
+          IMAGE_SIZE=`docker inspect -f "{{ .Size }}" ${{ steps.get-id.outputs.DOCKER_IMAGE }} | numfmt --to=si`;
+          echo "$IMAGE_SIZE container ${{ steps.get-id.outputs.DOCKER_IMAGE }}";
+
+      - name: Push the container image
+        run: docker push ${{ steps.get-id.outputs.DOCKER_IMAGE }}

.github/workflows/push-npm-from-container.yml

@@ -0,0 +1,84 @@
+name: Reusable container push workflow
+
+on:
+  workflow_call:
+
+env:
+  REPO: ${{ github.repository }}
+
+jobs:
+
+  push-npm:
+    runs-on: ubuntu-22.04
+    permissions:
+      # Give the default GITHUB_TOKEN write permission to commit and push the
+      # added or changed files to the repository.
+      contents: write
+
+    steps:
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 22
+        registry-url: ${{ vars.NPM_REGISTRY }}
+        token: ${{ secrets.NPM_TOKEN }}
+
+    - run: npm install -g pnpm
+
+    - uses: actions/checkout@v4
+
+    - id: get-id
+      name: Get a unique tag for this build
+      run: |
+        SHA=${{ github.sha }}; BRANCH_NAME=${{ github.base_ref || github.ref_name }};
+        BUILD_ID=$BRANCH_NAME-${SHA:0:8};
+        DOCKER_IMAGE=${{ vars.docker_repo2_registry }}/$REPO:$BUILD_ID;
+        echo "BUILD_ID=$BUILD_ID" >> "$GITHUB_OUTPUT";
+        echo "DOCKER_IMAGE=$DOCKER_IMAGE" >> "$GITHUB_OUTPUT";
+
+    - name: Print build id and image name
+      run: |
+        echo "BUILD_ID: ${{ steps.get-id.outputs.BUILD_ID }}"; 
+        echo "DOCKER_IMAGE: ${{ steps.get-id.outputs.DOCKER_IMAGE }}";
+
+    - name: Login to docker container registry
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ vars.docker_repo2_registry }}
+        username: ${{ secrets.docker_repo2_username }}
+        password: ${{ secrets.docker_repo2_password }}
+  
+    - name: Build the container image for npm build, with dependencies
+      run: |
+        docker build \
+          --build-arg BUILD_STEP=bundle \
+          --build-arg PUBLIC_BUILD_VERSION=${{ steps.get-id.outputs.BUILD_ID }} \
+          --file fab/d/actions-build.Dockerfile \
+          --tag ${{ steps.get-id.outputs.DOCKER_IMAGE }} \
+          .;
+
+    - name: Extract lib files
+      run: |
+        image=${{ steps.get-id.outputs.DOCKER_IMAGE }}
+        source_path=/src/lib
+        destination_path=lib
+
+        container_id=$(docker create "$image" "pnpm build_npm")
+        docker container start -a $container_id
+
+        docker cp "$container_id:$source_path" "$destination_path"
+        docker rm "$container_id"
+
+    - name: Increment package version and push
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        git config --global user.name 'bot-build'
+        git config --global user.email 'techbots+build@gmetri.com'
+
+        export N=`node -p require\(\'./package.json\'\).name` && echo $N
+        pnpm version patch --message "v%s: $N [CI SKIP]"
+
+        npm publish
+        
+        git push origin
+        git push --tags origin

.github/workflows/push-npm.yml

@@ -16,20 +16,20 @@ jobs:
       contents: write
 
     steps:
-    - uses: actions/checkout@v4
-      
     - uses: actions/setup-node@v4
       with:
         node-version: 22
         registry-url: ${{ vars.NPM_REGISTRY }}
         token: ${{ secrets.NPM_TOKEN }}
 
-    - name: Install npm dependencies
+    - run: npm install -g pnpm
-      run: |
-        npm install -g pnpm
-        pnpm install
 
-    - run: pnpm build_npm
+    - uses: actions/checkout@v4
+
+    - name: Install npm dependencies and build
+      run: |
+        pnpm install
+        pnpm build_npm
 
     - name: Increment package version and push
       env:

.github/workflows/push-s3-gcp.yml

@@ -0,0 +1,86 @@
+name: Docker Image CI
+
+on:
+  workflow_call:
+
+# Org Secrets are available on push event. Not pull_request event.
+
+env:
+  REPO: ${{ github.repository }}
+  REPO_SHORT_NAME: ${{ github.event.repository.name }}
+
+jobs:
+
+  push-s3:
+    runs-on: ubuntu-22.04
+    steps:
+    - id: get-id
+      name: Get a unique tag for this build
+      run: |
+        SHA=${{ github.sha }}; BRANCH_NAME=${{ github.base_ref || github.ref_name }};
+        BUILD_ID=$BRANCH_NAME-${SHA:0:8};
+        DOCKER_IMAGE=${{ vars.GCP_DOCKER_REGISTRY }}/$REPO:$BUILD_ID;
+        echo "BUILD_ID=$BUILD_ID" >> "$GITHUB_OUTPUT";
+        echo "DOCKER_IMAGE=$DOCKER_IMAGE" >> "$GITHUB_OUTPUT";
+
+    - name: Print build id and image name
+      run: |
+        echo "BUILD_ID: ${{ steps.get-id.outputs.BUILD_ID }}"; 
+        echo "DOCKER_IMAGE: ${{ steps.get-id.outputs.DOCKER_IMAGE }}";
+
+    - uses: actions/checkout@v4
+
+    # ✅ 1) Auth to GCP (this is where your SA key is used)
+    - name: Auth to GCP
+      uses: google-github-actions/auth@v2
+      with:
+        # using your existing secret that contains the SA JSON
+        credentials_json: ${{ secrets.GCP_SA_KEY }}
+
+      # ✅ 2) Install gcloud (no creds here)
+    - name: Set up gcloud
+      uses: google-github-actions/setup-gcloud@v2
+      with:
+        project_id: ${{ vars.GCP_PROJECT_ID }}
+        export_default_credentials: true
+
+    - name: Configure Docker for GAR
+      run: |
+        gcloud auth configure-docker ${{vars.GCP_REGION}}-docker.pkg.dev
+
+
+    - name: Build the container image for bundle step
+      run: |
+        docker build \
+          --build-arg BUILD_STEP=bundle \
+          --build-arg PUBLIC_BUILD_VERSION=${{ steps.get-id.outputs.BUILD_ID }} \
+          --file fab/d/actions-build.Dockerfile \
+          --tag ${{ steps.get-id.outputs.DOCKER_IMAGE }} \
+          .;
+    
+    - name: Extract cloud files
+      run: |
+        image=${{ steps.get-id.outputs.DOCKER_IMAGE }}
+        source_path=/cloud
+        destination_path=cloud
+
+        container_id=$(docker create "$image")
+        docker cp "$container_id:$source_path" "$destination_path"
+        docker rm "$container_id"
+
+        echo "Running: ls $destination_path"
+        ls $destination_path
+
+    - name: Upload cloud files
+      uses: https://git.gmetri.io/gmetribin/aws-cli-action@v1.0.0
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.aws_access_key_id }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.aws_secret_access_key }}
+        AWS_DEFAULT_REGION: ${{ vars.aws_default_region }}
+      with:
+        args: >
+          s3 cp \
+          --recursive \
+          --cache-control max-age=31536000\
+          --storage-class 'STANDARD_IA' \
+          cloud/ s3://${{ vars.aws_upload_bucket }}/${{ env.REPO_SHORT_NAME }}/${{ steps.get-id.outputs.BUILD_ID }}

.gitignore

@@ -129,4 +129,4 @@ dist
 .yarn/build-state.yml
 .yarn/install-state.gz
 .pnp.*
-
+.idea/

README-actions.md

@@ -51,22 +51,6 @@ https://github.com/orgs/community/discussions/26625#discussioncomment-3252582
 
 https://anchorecommunity.discourse.group/t/how-to-act-on-go-module-vulnerabilities/186/2
 
-Within the image:
-```bash
-curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b ./bin
-
-#To check vulnerabilities
-grypd .
-#To save detailed output
-grype $MY_IMAGE -o json > vuln.json
-#OR
-grype . -o json > vuln.json
-
-#To explain the issue:
-cat vuln.json | grype explain --id CVE-2023-24537
-cat vuln2.json | grype explain --id CVE-2023-45853
-```
-
 ## Getting the word "main" (branch name) during builds
 
 If the build (was triggered by) is a merge of a pull request, GITHUB_BASE_REF will contain main. 

repo-template/baseimage-repo/base-build-image.yml

@@ -1,18 +0,0 @@
-#For use in base images repos like basin/baseimages
-name: Docker Image CI
-
-on: 
-  push: # Build on all pushes
-  schedule: # Also build on all Fridays
-    - cron: "30 6 * * 5" #Every Friday@12 NOON IST (6:30 GMT)
-  # Cron: Minute(0-59) Hour(0-23) DayOfMonth(1-31) MonthOfYear(1-12) DayOfWeek(0-6)
-
-jobs:
-
-#Assumes the dockerfile to be at ./context/Dockerfile and context ./context
-  reuse-base-build-image:
-    uses: gmetribin/build-tools/.github/workflows/base-build-image.yml@v1.1.2
-    secrets: inherit
-    with:
-      image_tag: ${{ github.ref_name }}-v1 #Generally becomes basin:node-22-dev-v1
-      #To be updated in the code repo as per requirement

repo-template/node-based-repo/.github/workflows/base-workflow.yml

@@ -1,17 +0,0 @@
-name: Base Push Workflow
-
-on:
-  workflow_dispatch:
-  schedule: # Also build on all Fridays
-  - cron: "30 7 * * 5" #Every Friday@1PM IST (7:30 GMT)
-  # Cron: Minute(0-59) Hour(0-23) DayOfMonth(1-31) MonthOfYear(1-12) DayOfWeek(0-6)
-
-jobs:
-
-#Builds ./fab/d/actions-base.Dockerfile
-  cron-container-base:
-    uses: gmetribin/build-tools/.github/workflows/cron-container-base.yml@v1.1.2
-    secrets: inherit
-    with:
-      image_tag: base-v1 #To be updated in the code repo as per requirement
-      #Update the build image to use the same base tag

repo-template/node-based-repo/.github/workflows/pr-workflow.yml

@@ -1,13 +0,0 @@
-name: Pull Request Workflow
-
-on:
-  pull_request:
-    branches:
-      - main
-
-jobs:
-
-  #Runs pnpm lint and pnpm check
-  lint-and-check:
-    uses: gmetribin/build-tools/.github/workflows/pr-lint-and-check.yml@v1.1.4
-    secrets: inherit

repo-template/node-based-repo/.github/workflows/push-workflow.yml

@@ -1,40 +0,0 @@
-name: Push Workflow
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  #Runs `pnpm build_npm`
-  push-npm:
-    uses: gmetribin/build-tools/.github/workflows/push-npm.yml@v1.1.4
-    secrets: inherit
-
-  #Runs code vulnerability scan after `pnpm install`
-  code-scan:
-    uses: gmetribin/build-tools/.github/workflows/push-code-scan.yml@v1.1.4
-    secrets: inherit
-
-  #Builds ./fab/d/actions-build.Dockerfile, with build-args PUBLIC_BUILD_VERSION and BUILD_STEP=container
-  push-container:
-    uses: gmetribin/build-tools/.github/workflows/push-container.yml@v1.1.4
-    secrets: inherit
-
-  #Builds ./fab/d/actions-build.Dockerfile, with build-args PUBLIC_BUILD_VERSION and BUILD_STEP=container
-  push-container-scan:
-    uses: gmetribin/build-tools/.github/workflows/push-container-scan.yml@v1.1.4
-    secrets: inherit
-
-  #Expects the files to be sent to S3 to be placed at /cloud folder in the docker
-  #Builds ./fab/d/actions-build.Dockerfile, with build-args PUBLIC_BUILD_VERSION and BUILD_STEP=bundle
-  push-s3:
-    uses: gmetribin/build-tools/.github/workflows/push-s3.yml@v1.1.4
-    secrets: inherit
-
-  cs-update-trigger:
-    uses: gmetribin/deploy-tools/.github/workflows/cs-update-trigger.yml@v1.1.22
-    needs: [push-container]
-    secrets: inherit
-    with:
-      deploy_repo: gmetrivr/cs-dt #Update as per the repo group

repo-template/node-based-repo/d/actions-base.Dockerfile

@@ -1,19 +0,0 @@
-FROM repo2.hub.gmetri.io/gmetrivr/basin:node-22-dev-v2
-
-USER root
-
-RUN apt-get update \
-    && apt-get install -y wget gnupg2 lsb-release \
-    && sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \
-    && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
-    && apt-get -y update \
-	&& apt-get install -y --no-install-recommends \
-		postgresql-client-14 \
-    && apt-get clean all \
-    && rm -rf /var/lib/apt/lists/*
-
-USER node
-
-# No need to install dependencies in this step
-# COPY --chown=1000 ./package.json ./pnpm-lock.yaml .npmrc /src/
-# RUN pnpm install;

repo-template/node-based-repo/d/actions-build.Dockerfile

@@ -1,36 +0,0 @@
-FROM repo2.hub.gmetri.io/gmetrivr/basin:node-22-dev-v2
-
-#Remove old code
-# RUN find . -maxdepth 1 ! \( -name node_modules -o -name .pnpm-store \) -exec rm -rf "{}" \;
-
-#Replace with new code (node_modules and .pnpm_store are in dockerignore)
-COPY --chown=1000:1000 . /src
-
-#PUBLIC_BUILD_VERSION contains the unique build id for this image
-ARG PUBLIC_BUILD_VERSION
-ENV PUBLIC_BUILD_VERSION=$PUBLIC_BUILD_VERSION
-
-#BUILD_STEP is bundle or container. bundle for pushing to s3, container for docker image.
-ARG BUILD_STEP
-ENV BUILD_STEP=$BUILD_STEP
-
-RUN if [ "$BUILD_STEP" = "bundle" ]; then \
-      echo "BUNDLE version $PUBLIC_BUILD_VERSION"; \
-      pnpm install && \
-      pnpm build; \
-    elif [ "$BUILD_STEP" = "container" ]; then \
-      echo "CONTAINER version $PUBLIC_BUILD_VERSION"; \
-      pnpm install --production && \
-      pnpm prune && \
-      pnpm store prune; \
-    fi;
-
-#For single step:
-# RUN if [ "$BUILD_STEP" = "container" ] || [ "$BUILD_STEP" = "bundle" ]; then \
-#       echo "BUILD version $PUBLIC_BUILD_VERSION"; \
-#       pnpm install && \
-#       pnpm build-storybook; \
-#     fi
-
-EXPOSE 4225
-CMD ["pnpm", "start-server"]