From b58a4e21c6a523516a4c6d5d108a28281e9b7814 Mon Sep 17 00:00:00 2001 From: Nick Fyson Date: Mon, 27 Sep 2021 20:32:30 +0100 Subject: [PATCH 1/9] start validating code-scanning workflows --- script/validate-data/index.ts | 2 +- script/validate-data/settings.json | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/script/validate-data/index.ts b/script/validate-data/index.ts index 8413653..8153936 100755 --- a/script/validate-data/index.ts +++ b/script/validate-data/index.ts @@ -49,7 +49,7 @@ async function checkWorkflows(folders: string[]): Promise }); for (const e of dir) { - if (e.isFile()) { + if (e.isFile() && extname(e.name) === ".yml") { const fileType = basename(e.name, extname(e.name)) const workflowFilePath = join(folder, e.name); diff --git a/script/validate-data/settings.json b/script/validate-data/settings.json index f7c08cf..01092cc 100644 --- a/script/validate-data/settings.json +++ b/script/validate-data/settings.json @@ -2,6 +2,7 @@ "folders": [ "../../ci", "../../automation", - "../../deployments" + "../../deployments", + "../../code-scanning" ] } \ No newline at end of file From 70655750b2798ee5171c044e10795b992e27ee6f Mon Sep 17 00:00:00 2001 From: Nick Fyson Date: Tue, 28 Sep 2021 09:37:43 +0100 Subject: [PATCH 2/9] check for yml and yaml extensions --- script/validate-data/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/validate-data/index.ts b/script/validate-data/index.ts index 8153936..84518d7 100755 --- a/script/validate-data/index.ts +++ b/script/validate-data/index.ts @@ -49,7 +49,7 @@ async function checkWorkflows(folders: string[]): Promise }); for (const e of dir) { - if (e.isFile() && extname(e.name) === ".yml") { + if (e.isFile() && [".yml", ".yaml"].includes(extname(e.name))) { const fileType = basename(e.name, extname(e.name)) const workflowFilePath = join(folder, e.name); From f0b8c8ad72c009ca3093b9919e70a6325b49aa44 Mon Sep 17 00:00:00 2001 From: Gary Houbre Date: Tue, 28 Sep 2021 11:41:17 +0200 Subject: [PATCH 3/9] Starter workflow Symfony (#1069) * Add Symfony to starter Workflow * Added Properties from Symfony * Update symfony.yml * Update symfony.yml * Update symfony.yml * Fix Wrong Configuration * Review and fixing * Update Symfony Properties Category Co-authored-by: Ashwin Sangem --- ci/properties/symfony.properties.json | 10 +++++++ ci/symfony.yml | 39 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 ci/properties/symfony.properties.json create mode 100644 ci/symfony.yml diff --git a/ci/properties/symfony.properties.json b/ci/properties/symfony.properties.json new file mode 100644 index 0000000..ea9a950 --- /dev/null +++ b/ci/properties/symfony.properties.json @@ -0,0 +1,10 @@ +{ + "name": "Symfony", + "description": "Test a Symfony project.", + "iconName": "php", + "categories": [ + "Continuous integration", + "PHP", + "Symfony" + ] +} \ No newline at end of file diff --git a/ci/symfony.yml b/ci/symfony.yml new file mode 100644 index 0000000..7d1ca74 --- /dev/null +++ b/ci/symfony.yml @@ -0,0 +1,39 @@ +name: Symfony + +on: + push: + branches: [ $default-branch ] + pull_request: + branches: [ $default-branch ] + +jobs: + symfony-tests: + runs-on: ubuntu-latest + steps: + # To automatically get bug fixes and new Php versions for shivammathur/setup-php, + # change this to (see https://github.com/shivammathur/setup-php#bookmark-versioning): + # uses: shivammathur/setup-php@v2 + - uses: shivammathur/setup-php@2cb9b829437ee246e9b3cac53555a39208ca6d28 + with: + php-version: '8.0' + - uses: actions/checkout@v2 + - name: Copy .env.test.local + run: php -r "file_exists('.env.test.local') || copy('.env.test', '.env.test.local');" + - name: Cache Composer packages + id: composer-cache + uses: actions/cache@v2 + with: + path: vendor + key: ${{ runner.os }}-php-${{ hashFiles('**/composer.lock') }} + restore-keys: | + ${{ runner.os }}-php- + - name: Install Dependencies + run: composer install -q --no-ansi --no-interaction --no-scripts --no-progress --prefer-dist + - name: Create Database + run: | + mkdir -p data + touch data/database.sqlite + - name: Execute tests (Unit and Feature tests) via PHPUnit + env: + DATABASE_URL: sqlite:///%kernel.project_dir%/data/database.sqlite + run: vendor/bin/phpunit From 7f30309ccedb0e3dee186e0ee58c232752a78e24 Mon Sep 17 00:00:00 2001 From: Fernando de Oliveira <5161098+fedeoliv@users.noreply.github.com> Date: Wed, 29 Sep 2021 02:02:01 -0300 Subject: [PATCH 4/9] Azure Data Factory CI starter workflow (#1111) * Azure Data Factory CI starter workflow * fix: data factory starter categories * fix: checkout step formatting * fix: data-factory-export targeting latest version * feature: latest adf validate and export versions * feature: Azure Data Factory tech_stack category for CI starter Co-authored-by: Fernando de Oliveira <5161098+fernandoBRS@users.noreply.github.com> --- ci/azure-data-factory.yml | 47 +++++++++++++++++++ .../azure-data-factory.properties.json | 7 +++ icons/azure-data-factory.svg | 1 + 3 files changed, 55 insertions(+) create mode 100644 ci/azure-data-factory.yml create mode 100644 ci/properties/azure-data-factory.properties.json create mode 100644 icons/azure-data-factory.svg diff --git a/ci/azure-data-factory.yml b/ci/azure-data-factory.yml new file mode 100644 index 0000000..776e250 --- /dev/null +++ b/ci/azure-data-factory.yml @@ -0,0 +1,47 @@ +# Sample workflow to validate Azure Data Factory resources and export its ARM template as an artifact +# Note: Ensure you have the following package.json in the same directory of your ADF resources + +# { +# "scripts":{ +# "build":"node node_modules/@microsoft/azure-data-factory-utilities/lib/index" +# }, +# "dependencies":{ +# "@microsoft/azure-data-factory-utilities":"^0.1.5" +# } +# } + +name: Data Factory CI + +on: + pull_request: + branches: [ $default-branch, $protected-branches ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Code checkout + uses: actions/checkout@v2 + + - name: Validate + uses: Azure/data-factory-validate-action@v1.1.3 + # with: + # path: # replace by the folder that contains the Data Factory resources and the package.json + + # Generate the ARM template into the destination folder, which is the same as selecting "Publish" from the UX. + # The ARM template generated isn't published to the live version of the factory. + - name: Export ARM Template + id: export + uses: Azure/data-factory-export-action@v1.1.0 + # with: + # path: # replace by the folder that contains the Data Factory resources and the package.json + + - name: Publish artifact + uses: actions/upload-artifact@v2 + with: + name: Data Factory package + path: ${{ steps.export.outputs.arm-template-directory }} + if-no-files-found: error diff --git a/ci/properties/azure-data-factory.properties.json b/ci/properties/azure-data-factory.properties.json new file mode 100644 index 0000000..f1e7f5b --- /dev/null +++ b/ci/properties/azure-data-factory.properties.json @@ -0,0 +1,7 @@ +{ + "name": "Azure Data Factory", + "creator": "Microsoft Azure", + "description": "Build and validate Azure Data Factory resources", + "iconName": "azure-data-factory", + "categories": ["Continuous integration", "Azure Data Factory"] +} diff --git a/icons/azure-data-factory.svg b/icons/azure-data-factory.svg new file mode 100644 index 0000000..2237336 --- /dev/null +++ b/icons/azure-data-factory.svg @@ -0,0 +1 @@ +Icon-databases-126 \ No newline at end of file From 8c91a4c02f685e9ef8bf91b40cc51016d7b7e090 Mon Sep 17 00:00:00 2001 From: Sheldon Warkentin Date: Wed, 29 Sep 2021 13:45:57 -0600 Subject: [PATCH 5/9] Remoev mention of trial for Mayhem for API A free plan is now in place with a professional trial that may be opted into afterward. --- code-scanning/mayhem-for-api.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code-scanning/mayhem-for-api.yml b/code-scanning/mayhem-for-api.yml index 0aab0b4..59d66a0 100644 --- a/code-scanning/mayhem-for-api.yml +++ b/code-scanning/mayhem-for-api.yml @@ -10,7 +10,7 @@ # To use this workflow, you will need to: # # 1. Create a Mayhem for API account at -# https://mayhem4api.forallsecure.com/signup (30-day free trial) +# https://mayhem4api.forallsecure.com/signup # # 2. Create a service account token `mapi organization service-account create # ` From 6b053712bee09ee0862f8768f1810139cc3a95c4 Mon Sep 17 00:00:00 2001 From: Ashwin Sangem Date: Thu, 30 Sep 2021 07:31:43 +0530 Subject: [PATCH 6/9] Added dockerfile to relevant CD template categories. (#1136) * Added Dockerfile to Category for relevant CD templates. * Update terraform.properties.json --- deployments/properties/alibabacloud.properties.json | 2 +- deployments/properties/google.properties.json | 2 +- deployments/properties/ibm.properties.json | 2 +- deployments/properties/tencent.properties.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deployments/properties/alibabacloud.properties.json b/deployments/properties/alibabacloud.properties.json index 1ce285f..d416d88 100644 --- a/deployments/properties/alibabacloud.properties.json +++ b/deployments/properties/alibabacloud.properties.json @@ -3,5 +3,5 @@ "description": "Deploy a container to Alibaba Cloud Container Service for Kubernetes (ACK).", "creator": "Alibaba Cloud", "iconName": "alibabacloud", - "categories": ["Deployment"] + "categories": ["Deployment", "Dockerfile"] } diff --git a/deployments/properties/google.properties.json b/deployments/properties/google.properties.json index c7f216f..6318106 100644 --- a/deployments/properties/google.properties.json +++ b/deployments/properties/google.properties.json @@ -3,5 +3,5 @@ "description": "Build a docker container, publish it to Google Container Registry, and deploy to GKE.", "creator": "Google Cloud", "iconName": "googlegke", - "categories": ["Deployment"] + "categories": ["Deployment", "Dockerfile"] } \ No newline at end of file diff --git a/deployments/properties/ibm.properties.json b/deployments/properties/ibm.properties.json index 4219690..8e5b047 100644 --- a/deployments/properties/ibm.properties.json +++ b/deployments/properties/ibm.properties.json @@ -3,5 +3,5 @@ "description": "Build a docker container, publish it to IBM Cloud Container Registry, and deploy to IBM Cloud Kubernetes Service.", "creator": "IBM", "iconName": "ibm", - "categories": ["Deployment"] + "categories": ["Deployment", "Dockerfile"] } \ No newline at end of file diff --git a/deployments/properties/tencent.properties.json b/deployments/properties/tencent.properties.json index 32d0da7..df18356 100644 --- a/deployments/properties/tencent.properties.json +++ b/deployments/properties/tencent.properties.json @@ -3,5 +3,5 @@ "description": "This workflow will build a docker container, publish and deploy it to Tencent Kubernetes Engine (TKE).", "creator": "Tencent Cloud", "iconName": "tencentcloud", - "categories": ["Deployment"] + "categories": ["Deployment", "Dockerfile"] } \ No newline at end of file From 02d91c6ccfbd5c7bfe25e397dfe80882a725b364 Mon Sep 17 00:00:00 2001 From: Aparna Ravindra <82894348+aparna-ravindra@users.noreply.github.com> Date: Thu, 30 Sep 2021 10:19:20 +0530 Subject: [PATCH 7/9] checking for allowed category in validate-data script (#1131) * checking for allowed category * Update index.ts --- ci/properties/super-linter.properties.json | 2 +- script/validate-data/index.ts | 13 ++++++++----- script/validate-data/settings.json | 6 ++++++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/ci/properties/super-linter.properties.json b/ci/properties/super-linter.properties.json index e070300..0be52e1 100644 --- a/ci/properties/super-linter.properties.json +++ b/ci/properties/super-linter.properties.json @@ -2,5 +2,5 @@ "name": "Super Linter - Run Linters for several languages", "description": "Run linters for several languages on your code base for changed files", "iconName": "octicon check-circle", - "categories": ["code-quality", "code-review"] + "categories": ["Continuous integration", "code-quality", "code-review"] } diff --git a/script/validate-data/index.ts b/script/validate-data/index.ts index 84518d7..7dce3d1 100755 --- a/script/validate-data/index.ts +++ b/script/validate-data/index.ts @@ -40,7 +40,7 @@ const propertiesSchema = { } } -async function checkWorkflows(folders: string[]): Promise { +async function checkWorkflows(folders: string[], allowed_categories: string[]): Promise { const result: WorkflowWithErrors[] = [] const workflow_template_names = new Set() for (const folder of folders) { @@ -55,7 +55,7 @@ async function checkWorkflows(folders: string[]): Promise const workflowFilePath = join(folder, e.name); const propertiesFilePath = join(folder, "properties", `${fileType}.properties.json`) - const workflowWithErrors = await checkWorkflow(workflowFilePath, propertiesFilePath); + const workflowWithErrors = await checkWorkflow(workflowFilePath, propertiesFilePath, allowed_categories); if(workflowWithErrors.name && workflow_template_names.size == workflow_template_names.add(workflowWithErrors.name).size) { workflowWithErrors.errors.push(`Workflow template name "${workflowWithErrors.name}" already exists`) } @@ -69,13 +69,12 @@ async function checkWorkflows(folders: string[]): Promise return result; } -async function checkWorkflow(workflowPath: string, propertiesPath: string): Promise { +async function checkWorkflow(workflowPath: string, propertiesPath: string, allowed_categories: string[]): Promise { let workflowErrors: WorkflowWithErrors = { id: workflowPath, name: null, errors: [] } - try { const workflowFileContent = await fs.readFile(workflowPath, "utf8"); safeLoad(workflowFileContent); // Validate yaml parses without error @@ -105,6 +104,10 @@ async function checkWorkflow(workflowPath: string, propertiesPath: string): Prom } } + if (!workflowPath.endsWith("blank.yml") && (!properties.categories || + !properties.categories.some(category => allowed_categories.some(ac => ac.toLowerCase() == category.toLowerCase())))) { + workflowErrors.errors.push(`Workflow does not contain at least one allowed category - ${allowed_categories}`) + } } catch (e) { workflowErrors.errors.push(e.toString()) } @@ -115,7 +118,7 @@ async function checkWorkflow(workflowPath: string, propertiesPath: string): Prom try { const settings = require("./settings.json"); const erroredWorkflows = await checkWorkflows( - settings.folders + settings.folders, settings.allowed_categories ) if (erroredWorkflows.length > 0) { diff --git a/script/validate-data/settings.json b/script/validate-data/settings.json index 01092cc..ce89e36 100644 --- a/script/validate-data/settings.json +++ b/script/validate-data/settings.json @@ -4,5 +4,11 @@ "../../automation", "../../deployments", "../../code-scanning" + ], + "allowed_categories" : [ + "Continuous integration", + "Deployment", + "Code Scanning", + "Automation" ] } \ No newline at end of file From 7b9e3b68582d04b65cb955fb75c48342450b7f3b Mon Sep 17 00:00:00 2001 From: Ashwin Sangem Date: Fri, 1 Oct 2021 18:50:08 +0530 Subject: [PATCH 8/9] Revert "Azure Data Factory CI starter workflow (#1111)" (#1146) This reverts commit 7f30309ccedb0e3dee186e0ee58c232752a78e24. --- ci/azure-data-factory.yml | 47 ------------------- .../azure-data-factory.properties.json | 7 --- icons/azure-data-factory.svg | 1 - 3 files changed, 55 deletions(-) delete mode 100644 ci/azure-data-factory.yml delete mode 100644 ci/properties/azure-data-factory.properties.json delete mode 100644 icons/azure-data-factory.svg diff --git a/ci/azure-data-factory.yml b/ci/azure-data-factory.yml deleted file mode 100644 index 776e250..0000000 --- a/ci/azure-data-factory.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Sample workflow to validate Azure Data Factory resources and export its ARM template as an artifact -# Note: Ensure you have the following package.json in the same directory of your ADF resources - -# { -# "scripts":{ -# "build":"node node_modules/@microsoft/azure-data-factory-utilities/lib/index" -# }, -# "dependencies":{ -# "@microsoft/azure-data-factory-utilities":"^0.1.5" -# } -# } - -name: Data Factory CI - -on: - pull_request: - branches: [ $default-branch, $protected-branches ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Code checkout - uses: actions/checkout@v2 - - - name: Validate - uses: Azure/data-factory-validate-action@v1.1.3 - # with: - # path: # replace by the folder that contains the Data Factory resources and the package.json - - # Generate the ARM template into the destination folder, which is the same as selecting "Publish" from the UX. - # The ARM template generated isn't published to the live version of the factory. - - name: Export ARM Template - id: export - uses: Azure/data-factory-export-action@v1.1.0 - # with: - # path: # replace by the folder that contains the Data Factory resources and the package.json - - - name: Publish artifact - uses: actions/upload-artifact@v2 - with: - name: Data Factory package - path: ${{ steps.export.outputs.arm-template-directory }} - if-no-files-found: error diff --git a/ci/properties/azure-data-factory.properties.json b/ci/properties/azure-data-factory.properties.json deleted file mode 100644 index f1e7f5b..0000000 --- a/ci/properties/azure-data-factory.properties.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "Azure Data Factory", - "creator": "Microsoft Azure", - "description": "Build and validate Azure Data Factory resources", - "iconName": "azure-data-factory", - "categories": ["Continuous integration", "Azure Data Factory"] -} diff --git a/icons/azure-data-factory.svg b/icons/azure-data-factory.svg deleted file mode 100644 index 2237336..0000000 --- a/icons/azure-data-factory.svg +++ /dev/null @@ -1 +0,0 @@ -Icon-databases-126 \ No newline at end of file From 596b345944af4fbcae1bdcfaca339abd10bd82ed Mon Sep 17 00:00:00 2001 From: Sarah Edwards Date: Fri, 1 Oct 2021 12:07:03 -0700 Subject: [PATCH 9/9] use env variables for user-set values (#1117) Co-authored-by: Josh Gross --- deployments/aws.yml | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/deployments/aws.yml b/deployments/aws.yml index 8b10116..6a6643c 100644 --- a/deployments/aws.yml +++ b/deployments/aws.yml @@ -5,29 +5,40 @@ # # 1. Create an ECR repository to store your images. # For example: `aws ecr create-repository --repository-name my-ecr-repo --region us-east-2`. -# Replace the value of `ECR_REPOSITORY` in the workflow below with your repository's name. -# Replace the value of `aws-region` in the workflow below with your repository's region. +# Replace the value of the `ECR_REPOSITORY` environment variable in the workflow below with your repository's name. +# Replace the value of the `AWS_REGION` environment variable in the workflow below with your repository's region. # # 2. Create an ECS task definition, an ECS cluster, and an ECS service. # For example, follow the Getting Started guide on the ECS console: # https://us-east-2.console.aws.amazon.com/ecs/home?region=us-east-2#/firstRun -# Replace the values for `service` and `cluster` in the workflow below with your service and cluster names. +# Replace the value of the `ECS_SERVICE` environment variable in the workflow below with the name you set for the Amazon ECS service. +# Replace the value of the `ECS_CLUSTER` environment variable in the workflow below with the name you set for the cluster. # # 3. Store your ECS task definition as a JSON file in your repository. # The format should follow the output of `aws ecs register-task-definition --generate-cli-skeleton`. -# Replace the value of `task-definition` in the workflow below with your JSON file's name. -# Replace the value of `container-name` in the workflow below with the name of the container +# Replace the value of the `ECS_TASK_DEFINITION` environment variable in the workflow below with the path to the JSON file. +# Replace the value of the `CONTAINER_NAME` environment variable in the workflow below with the name of the container # in the `containerDefinitions` section of the task definition. # # 4. Store an IAM user access key in GitHub Actions secrets named `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. # See the documentation for each action used below for the recommended IAM policies for this IAM user, # and best practices on handling the access key credentials. +name: Deploy to Amazon ECS + on: release: types: [created] -name: Deploy to Amazon ECS +env: + AWS_REGION: MY_AWS_REGION # set this to your preferred AWS region, e.g. us-west-1 + ECR_REPOSITORY: MY_ECR_REPOSITORY # set this to your Amazon ECR repository name + ECS_SERVICE: MY_ECS_SERVICE # set this to your Amazon ECS service name + ECS_CLUSTER: MY_ECS_CLUSTER # set this to your Amazon ECS cluster name + ECS_TASK_DEFINITION: MY_ECS_TASK_DEFINITION # set this to the path to your Amazon ECS task definition + # file, e.g. .aws/task-definition.json + CONTAINER_NAME: MY_CONTAINER_NAME # set this to the name of the container in the + # containerDefinitions section of your task definition jobs: deploy: @@ -44,7 +55,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 + aws-region: ${{ env.AWS_REGION }} - name: Login to Amazon ECR id: login-ecr @@ -54,7 +65,6 @@ jobs: id: build-image env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: my-ecr-repo IMAGE_TAG: ${{ github.sha }} run: | # Build a docker container and @@ -68,14 +78,14 @@ jobs: id: task-def uses: aws-actions/amazon-ecs-render-task-definition@v1 with: - task-definition: task-definition.json - container-name: sample-app + task-definition: ${{ env.ECS_TASK_DEFINITION }} + container-name: ${{ env.CONTAINER_NAME }} image: ${{ steps.build-image.outputs.image }} - name: Deploy Amazon ECS task definition uses: aws-actions/amazon-ecs-deploy-task-definition@v1 with: task-definition: ${{ steps.task-def.outputs.task-definition }} - service: sample-app-service - cluster: default + service: ${{ env.ECS_SERVICE }} + cluster: ${{ env.ECS_CLUSTER }} wait-for-service-stability: true