diff options
| -rw-r--r-- | .github/PULL_REQUEST_TEMPLATE.md | 47 | ||||
| -rw-r--r-- | .github/labeler.yml | 9 | ||||
| -rw-r--r-- | .github/reviewers.yml | 3 | ||||
| -rw-r--r-- | .github/workflows/auto-author-assign.yml | 27 | ||||
| -rw-r--r-- | .github/workflows/codeql-analysis.yml | 6 | ||||
| -rw-r--r-- | .github/workflows/mergifyio_backport.yml | 22 | ||||
| -rw-r--r-- | .github/workflows/pr-conflicts.yml | 18 | ||||
| -rw-r--r-- | .github/workflows/pull-request-labels.yml | 19 | ||||
| -rw-r--r-- | .github/workflows/pull-request-management.yml | 25 | ||||
| -rw-r--r-- | .github/workflows/pull-request-message-check.yml | 23 | ||||
| -rw-r--r-- | .github/workflows/stale.yml | 22 | ||||
| -rw-r--r-- | .github/workflows/unused-imports.yml | 22 | ||||
| -rw-r--r-- | README.md | 60 | ||||
| -rwxr-xr-x | scripts/check-pr-title-and-commit-messages.py | 51 | ||||
| -rw-r--r-- | scripts/override-default | 140 | ||||
| -rw-r--r-- | scripts/transclude-template | 50 |
16 files changed, 540 insertions, 4 deletions
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..4c8266c --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,47 @@ +<!-- All PR should follow this template to allow a clean and transparent review --> +<!-- Text placed between these delimiters is considered a comment and is not rendered --> + +## Change Summary +<!--- Provide a general summary of your changes in the Title above --> + +## Types of changes +<!--- +What types of changes does your code introduce? Put an 'x' in all the boxes that apply. +NOTE: Markdown requires no leading or trailing whitespace inside the [ ] for checking +the box, please use [x] +--> +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Code style update (formatting, renaming) +- [ ] Refactoring (no functional changes) +- [ ] Other (please describe): + +## Related Task(s) +<!-- optional: Link to related other tasks on Phabricator. --> +<!-- * https://vyos.dev/Txxxx --> + +## Related PR(s) +<!-- Link here any PRs in other repositories that are required by this PR --> + +## Proposed changes +<!--- Describe your changes in detail --> + +## How to test +<!--- +Please describe in detail how you tested your changes. Include details of your testing +environment, and the tests you ran. When pasting configs, logs, shell output, backtraces, +and other large chunks of text, surround this text with triple backtics +``` +like this +``` +--> + +## Checklist: +<!--- Go over all the following points, and put an `x` in all the boxes that apply. --> +<!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! --> +<!--- The entire development process is outlined here: https://docs.vyos.io/en/latest/contributing/development.html --> +- [ ] I have read the [**CONTRIBUTING**](https://github.com/vyos/vyos-1x/blob/current/CONTRIBUTING.md) document +- [ ] I have linked this PR to one or more Phabricator Task(s) +- [ ] My commit headlines contain a valid Task id +- [ ] My change requires a change to the documentation +- [ ] I have updated the documentation accordingly diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 0000000..3a16d27 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,9 @@ +equuleus: + - any: + - base-branch: 'equuleus' +current: + - any: + - base-branch: 'current' +sagitta: + - any: + - base-branch: 'sagitta' diff --git a/.github/reviewers.yml b/.github/reviewers.yml new file mode 100644 index 0000000..a1647d2 --- /dev/null +++ b/.github/reviewers.yml @@ -0,0 +1,3 @@ +--- +"**/*": + - team: reviewers diff --git a/.github/workflows/auto-author-assign.yml b/.github/workflows/auto-author-assign.yml new file mode 100644 index 0000000..1a7f8ef --- /dev/null +++ b/.github/workflows/auto-author-assign.yml @@ -0,0 +1,27 @@ +name: "PR Triage" +on: + pull_request_target: + types: [opened, reopened, ready_for_review, locked] + +permissions: + pull-requests: write + +jobs: + # https://github.com/marketplace/actions/auto-author-assign + assign-author: + runs-on: ubuntu-latest + steps: + - name: "Assign Author to PR" + uses: toshimaru/auto-author-assign@v1.6.2 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + # https://github.com/shufo/auto-assign-reviewer-by-files + assign_reviewer: + runs-on: ubuntu-latest + steps: + - name: Request review based on files changes and/or groups the author belongs to + uses: shufo/auto-assign-reviewer-by-files@v1.1.4 + with: + token: ${{ secrets.PR_ACTION_ASSIGN_REVIEWERS }} + config: .github/reviewers.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 75f8a1b..7176702 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -41,7 +41,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{matrix.language}} config-file: ${{inputs.codeql-cfg-path}} @@ -50,7 +50,7 @@ jobs: # If this step fails, configure a build command manually using build-command input. This command will be executed in the corresponding step. - name: Autobuild if: ${{!inputs.build-command}} - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 - name: Manual build if: ${{inputs.build-command}} @@ -58,4 +58,4 @@ jobs: ${{inputs.build-command}} - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/mergifyio_backport.yml b/.github/workflows/mergifyio_backport.yml new file mode 100644 index 0000000..d9f863d --- /dev/null +++ b/.github/workflows/mergifyio_backport.yml @@ -0,0 +1,22 @@ +name: Mergifyio backport + +on: [issue_comment] + +jobs: + mergifyio_backport: + if: github.repository == 'vyos/vyos-1x' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions-ecosystem/action-regex-match@v2 + id: regex-match + with: + text: ${{ github.event.comment.body }} + regex: '@[Mm][Ee][Rr][Gg][Ii][Ff][Yy][Ii][Oo] backport ' + + - uses: actions-ecosystem/action-add-labels@v1 + if: ${{ steps.regex-match.outputs.match != '' }} + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + labels: backport diff --git a/.github/workflows/pr-conflicts.yml b/.github/workflows/pr-conflicts.yml new file mode 100644 index 0000000..2fd0bb4 --- /dev/null +++ b/.github/workflows/pr-conflicts.yml @@ -0,0 +1,18 @@ +name: "PR Conflicts checker" +on: + pull_request_target: + types: [synchronize] + +jobs: + Conflict_Check: + name: 'Check PR status: conflicts and resolution' + runs-on: ubuntu-latest + steps: + - name: check if PRs are dirty + uses: eps1lon/actions-label-merge-conflict@v3 + with: + dirtyLabel: "state: conflict" + removeOnDirtyLabel: "state: conflict resolved" + repoToken: "${{ secrets.GITHUB_TOKEN }}" + commentOnDirty: "This pull request has conflicts, please resolve those before we can evaluate the pull request." + commentOnClean: "Conflicts have been resolved. A maintainer will review the pull request shortly." diff --git a/.github/workflows/pull-request-labels.yml b/.github/workflows/pull-request-labels.yml new file mode 100644 index 0000000..31e1bd9 --- /dev/null +++ b/.github/workflows/pull-request-labels.yml @@ -0,0 +1,19 @@ +--- +name: Add pull request labels + +on: + pull_request_target: + branches: + - current + - equuleus + - sagitta + +jobs: + add-pr-label: + name: Add PR Labels + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/labeler@v5 diff --git a/.github/workflows/pull-request-management.yml b/.github/workflows/pull-request-management.yml new file mode 100644 index 0000000..34c9c45 --- /dev/null +++ b/.github/workflows/pull-request-management.yml @@ -0,0 +1,25 @@ +--- +name: Build Pull Request Package + +on: + pull_request: + branches: + - current + - sagitta + - equuleus + +jobs: + j2lint: + name: Validate j2 files + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + timeout-minutes: 2 + - name: Setup J2Lint + timeout-minutes: 2 + run: | + sudo pip install git+https://github.com/aristanetworks/j2lint.git@341b5d5db86e095b622f09770cb6367a1583620e + - name: Run J2lint + timeout-minutes: 2 + run: | + j2lint $GITHUB_WORKSPACE/data diff --git a/.github/workflows/pull-request-message-check.yml b/.github/workflows/pull-request-message-check.yml new file mode 100644 index 0000000..2487879 --- /dev/null +++ b/.github/workflows/pull-request-message-check.yml @@ -0,0 +1,23 @@ +--- +name: Check pull request message format + +on: + pull_request: + branches: + - current + - sagitta + - equuleus + +jobs: + check-pr-title: + name: Check pull request title + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + timeout-minutes: 2 + - name: Install the requests library + run: pip3 install requests + - name: Check the PR title + timeout-minutes: 2 + run: | + ./scripts/check-pr-title-and-commit-messages.py '${{ github.event.pull_request.url }}' diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..d21d151 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,22 @@ +name: "Issue and PR stale management" +on: + schedule: + - cron: "0 0 * * *" + +jobs: + stale: + runs-on: ubuntu-latest + if: github.repository == 'vyos/vyos-1x' + steps: + # Issue stale management + - uses: actions/stale@v6 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + days-before-stale: 90 + days-before-close: -1 + stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. The issue will be reviewed by a maintainer and may be closed' + stale-issue-label: 'state: stale' + exempt-issue-labels: 'state: accepted, state: in-progress' + stale-pr-message: 'This PR is stale because it has been open 30 days with no activity. The PR will be reviewed by a maintainer and may be closed' + stale-pr-label: 'state: stale' + exempt-pr-labels: 'state: accepted, state: in-progress' diff --git a/.github/workflows/unused-imports.yml b/.github/workflows/unused-imports.yml new file mode 100644 index 0000000..da57bd2 --- /dev/null +++ b/.github/workflows/unused-imports.yml @@ -0,0 +1,22 @@ +name: Check for unused imports using Pylint +on: + pull_request_target: + branches: + - current + - sagitta + +jobs: + Check-Unused-Imports: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: 3.11 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint + - name: Analysing the code with pylint + run: make unused-imports @@ -1 +1,59 @@ -# vyos-github-actions
\ No newline at end of file +# vyos-github-actions + +<!-- start title --> + +This keeps all the reusable github action workflows for vyos + +## codeql-analysis ## + +This reusable workflow performs codeql analysis on the invoking repo using given inputs. + +This performs below: + +- Checkout the code. +- Initialize codeql. This uses the input languages. Initializes for each language given in input. +- Build the code. Either using autobuild or manual build as per input. +- Analyze with codeql. + +_Usage_: + +```yaml +name: "Perform CodeQL Analysis" + +on: + push: + branches: [ "current", "sagitta", "equuleus" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "current" ] + schedule: + - cron: '22 10 * * 0' + +permissions: + actions: read + contents: read + security-events: write + +jobs: + codeql-analysis-call: + uses: vyos/vyos-github-actions/.github/workflows/codeql-analysis.yml@main + secrets: inherit + with: + languages: "['python']" +``` + +<!-- end usage --> +<!-- start inputs --> + +| **Input** | **Description** | **Default** | **Required** | +| ---------------------- | ---------------------------------------------------------------------------------------------- | ---------------| ------------- | +| **`languages`** | Languages for CodeQL check. Supported values are: 'cpp', 'csharp', 'go', 'java', 'javascript' | **['python']** | **false** | +| **`codeql-cfg-path`** | Path to a CodeQL config file | | **false** | +| **`build-command`** | Manual build command. The multiline syntax is supported | | **false** | + +<!-- end inputs --> +Referenece: +[Codeql Action](https://github.com/github/codeql-action) + + +Also see the [GitHub reusable workflows documentation](https://docs.github.com/en/actions/creating-actions/sharing-actions-and-workflows-from-your-private-repository) diff --git a/scripts/check-pr-title-and-commit-messages.py b/scripts/check-pr-title-and-commit-messages.py new file mode 100755 index 0000000..001f6cf --- /dev/null +++ b/scripts/check-pr-title-and-commit-messages.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import re +import sys +import time + +import requests + +# Use the same regex for PR title and commit messages for now +title_regex = r'^(([a-zA-Z\-_.]+:\s)?)T\d+:\s+[^\s]+.*' +commit_regex = title_regex + +def check_pr_title(title): + if not re.match(title_regex, title): + print("PR title '{}' does not match the required format!".format(title)) + print("Valid title example: T99999: make IPsec secure") + sys.exit(1) + +def check_commit_message(title): + if not re.match(commit_regex, title): + print("Commit title '{}' does not match the required format!".format(title)) + print("Valid title example: T99999: make IPsec secure") + sys.exit(1) + +if __name__ == '__main__': + if len(sys.argv) < 2: + print("Please specify pull request URL!") + sys.exit(1) + + # There seems to be a race condition that causes this scripts to receive + # an incomplete PR object that is missing certain fields, + # which causes temporary CI failures that require re-running the script + # + # It's probably better to add a small delay to prevent that + time.sleep(5) + + # Get the pull request object + pr = requests.get(sys.argv[1]).json() + if "title" not in pr: + print("The PR object does not have a title field!") + print("Did not receive a valid pull request object, please check the URL!") + sys.exit(1) + + check_pr_title(pr["title"]) + + # Get the list of commits + commits = requests.get(pr["commits_url"]).json() + for c in commits: + # Retrieve every individual commit and check its title + co = requests.get(c["url"]).json() + check_commit_message(co["commit"]["message"]) diff --git a/scripts/override-default b/scripts/override-default new file mode 100644 index 0000000..5058e79 --- /dev/null +++ b/scripts/override-default @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# +# override-default: preprocessor for XML interface definitions to interpret +# redundant entries (relative to path) with tag 'defaultValue' as an override +# directive. Must be called before build-command-templates, as the schema +# disallows redundancy. +# +# Copyright (C) 2021 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# + +# Use lxml xpath capability to find multiple elements with tag defaultValue +# relative to path; replace and remove to override the value. + +import sys +import glob +import logging +from copy import deepcopy +from lxml import etree + +debug = False + +logger = logging.getLogger(__name__) +logs_handler = logging.StreamHandler() +logger.addHandler(logs_handler) + +if debug: + logger.setLevel(logging.DEBUG) +else: + logger.setLevel(logging.INFO) + +def clear_empty_path(el): + # on the odd chance of interleaved comments + tmp = [l for l in el if isinstance(l.tag, str)] + if not tmp: + p = el.getparent() + p.remove(el) + clear_empty_path(p) + +def override_element(l: list): + """ + Allow multiple override elements; use the final one (in document order). + """ + if len(l) < 2: + logger.debug("passing list of single element to override_element") + return + + # assemble list of leafNodes of overriding defaultValues, for later removal + parents = [] + for el in l[1:]: + parents.append(el.getparent()) + + # replace element with final override + l[0].getparent().replace(l[0], l[-1]) + + # remove all but overridden element + for el in parents: + tmp = el.getparent() + tmp.remove(el) + clear_empty_path(tmp) + +def merge_remaining(l: list, elementtree): + """ + Merge (now) single leaf node containing 'defaultValue' with leaf nodes + of same path and no 'defaultValue'. + """ + for p in l: + p = p.split() + path_str = f'/interfaceDefinition/*' + path_list = [] + for i in range(len(p)): + path_list.append(f'[@name="{p[i]}"]') + path_str += '/children/*'.join(path_list) + rp = elementtree.xpath(path_str) + if len(rp) > 1: + for el in rp[1:]: + # in practice there will only be one child of the path, + # either defaultValue or Properties, since + # override_element() has already run + for child in el: + rp[0].append(deepcopy(child)) + tmp = el.getparent() + tmp.remove(el) + clear_empty_path(tmp) + +def collect_and_override(dir_name): + """ + Collect elements with defaultValue tag into dictionary indexed by name + attributes of ancestor path. + """ + for fname in glob.glob(f'{dir_name}/*.xml'): + tree = etree.parse(fname) + root = tree.getroot() + defv = {} + + xpath_str = '//defaultValue' + xp = tree.xpath(xpath_str) + + for element in xp: + ap = element.xpath('ancestor::*[@name]') + ap_name = [el.get("name") for el in ap] + ap_path_str = ' '.join(ap_name) + defv.setdefault(ap_path_str, []).append(element) + + for k, v in defv.items(): + if len(v) > 1: + logger.info(f"overriding default in path '{k}'") + override_element(v) + + to_merge = list(defv) + merge_remaining(to_merge, tree) + + revised_str = etree.tostring(root, encoding='unicode', pretty_print=True) + + with open(f'{fname}', 'w') as f: + f.write(revised_str) + +def main(): + if len(sys.argv) < 2: + logger.critical('Must specify XML directory!') + sys.exit(1) + + dir_name = sys.argv[1] + + collect_and_override(dir_name) + +if __name__ == '__main__': + main() diff --git a/scripts/transclude-template b/scripts/transclude-template new file mode 100644 index 0000000..5c6668a --- /dev/null +++ b/scripts/transclude-template @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# +# transclude-template: preprocessor for XML interface definitions to +# interpret #include statements to include nested XML fragments and +# snippets in documents. +# +# Copyright (C) 2021 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# + +import os +import re +import sys + +regexp = re.compile(r'^ *#include <(.+)>$') + +def parse_file(filename): + lines = "" + with open(filename, 'r') as f: + while True: + line = f.readline() + if line: + result = regexp.match(line) + if result: + lines += parse_file(os.path.join(directory, result.group(1))) + else: + lines += line + else: + return lines + +if __name__ == '__main__': + if len(sys.argv) < 2: + print('Must specify XML file!', file=sys.stderr) + sys.exit(1) + filename = sys.argv[1] + directory = os.path.dirname(os.path.abspath(filename)) + print(parse_file(filename)) + |