commit b8788c239efa6b5b94df62bf19c148ecabad5f04 Author: Binbim_ProMax Date: Sat Jul 4 07:59:04 2026 +0800 feat: SafeLine CE 9.3.9 with pro features - Remove telemetry: disabled all phone-home (installation notify, false positives, behavior tracking, upgrade tips) - Version branding: removed 'ce-' prefix, product name changed to '长亭雷池 WAF' - Detection engine enhancement: strict preset enables all 17 modules with aggressive configs - Multi-user RBAC: admin/operator/viewer roles with password + TOTP 2FA - Rate limiting: nginx limit_req per website (RPS, burst, action) - JS Challenge: browser fingerprint verification page with cookie-based bypass - Security fixes: removed InsecureSkipVerify, hardcoded credentials, NO_AUTH backdoor diff --git a/.github/ISSUE_TEMPLATE/bug-report.yaml b/.github/ISSUE_TEMPLATE/bug-report.yaml new file mode 100644 index 0000000..cc7996c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yaml @@ -0,0 +1,45 @@ +name: Bug +description: Report a bug +# Create a report to help us improve +title: "[Bug] " + +body: + - type: markdown + attributes: + value: | + Please search the [open issues](https://github.com/chaitin/SafeLine/issues) and [discussion](https://github.com/chaitin/SafeLine/discussions) for duplicate issue first. + The more information you share, the faster we can identify and fix the bug. + # Please check for duplicate issue first. + - type: textarea + id: Description + attributes: + label: What happened? + # Describe the bug + validations: + required: false + - type: textarea + id: Reproduce + attributes: + label: How we reproduce? + description: | + Reports cannot be reproduced will Most likely be closed. + # To Reproduce + value: | + 1. ... + 2. ... + 3. ... + - type: textarea + id: Expected + attributes: + label: Expected behavior + # placeholder: | + # Descript what you expected to happen. + # Expected behavior. Descript what you expected to happen. + - type: textarea + id: Errorlog + attributes: + label: Error log + placeholder: | + Paste the error logs if any. + # Expected behavior. Descript what you expected to happen. + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..311e263 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +blank_issues_enabled: false +contact_links: + - name: Discord + url: https://discord.gg/wyshSVuvxC + about: Ask questions and discuss with other SafeLine users in real time. + - name: Home page 官网 + url: https://waf.chaitin.com/ + about: Get feature descriptions, technical documentation and more information. 获取功能描述、技术文档和更多信息。 + - name: 绕过反馈 + url: https://stack.chaitin.com/security-challenge/safeline/index + about: Waf 绕过可在 CT Stack 安全挑战赛提交细节 \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature-request.yaml b/.github/ISSUE_TEMPLATE/feature-request.yaml new file mode 100644 index 0000000..371e2e1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yaml @@ -0,0 +1,28 @@ +name: Suggestion +# Feature request +description: New feature or improvements. +title: "[Suggestion] " + +body: + - type: markdown + attributes: + value: | + Please search the [open issues](https://github.com/chaitin/SafeLine/issues) and [discussion](https://github.com/chaitin/SafeLine/discussions) for duplicate issue first. + Please rise only one suggestion in an issue. + - type: textarea + id: solution + attributes: + label: What would you like to be added or improved? + # Describe the solution you'd like + # placeholder: | + # + - type: textarea + id: problem + attributes: + label: Why is it needed? + # Background and the specific problem that frustrates you + placeholder: | + Background and the problem that frustrates you + + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/other.yaml b/.github/ISSUE_TEMPLATE/other.yaml new file mode 100644 index 0000000..58e2e20 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/other.yaml @@ -0,0 +1,12 @@ +name: Other +description: Other issues such as Doc +body: + - type: markdown + attributes: + value: | + Please search the [open issues](https://github.com/chaitin/SafeLine/issues) and [discussion](https://github.com/chaitin/SafeLine/discussions) for duplicate issue first. + Please rise only one suggestion in an issue. + - type: textarea + id: content + attributes: + label: Content \ No newline at end of file diff --git a/.github/workflows/slmcp-docker.yml b/.github/workflows/slmcp-docker.yml new file mode 100644 index 0000000..b2c3d50 --- /dev/null +++ b/.github/workflows/slmcp-docker.yml @@ -0,0 +1,41 @@ +name: MCP Docker + +on: + push: + branches: + - main + tags: + - "v*" + paths: + - "mcp_server/**" + - ".github/workflows/slmcp-docker.yml" + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + platforms: linux/amd64,linux/arm64 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERIO_USERNAME }} + password: ${{ secrets.DOCKERIO_PASSWORD }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: ./mcp_server + push: true + platforms: linux/amd64,linux/arm64 + tags: | + chaitin/safeline-mcp:latest + chaitin/safeline-mcp:${{ github.ref_name }} + cache-from: type=registry,ref=chaitin/safeline-mcp:buildcache + cache-to: type=registry,ref=chaitin/safeline-mcp:buildcache,mode=max diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5ee874a --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +.env +*.env +*.db +*.db-* +node_modules/ +vendor/ +build/ +dist/ +*.exe +*.log +logs/ +.idea/ +.vscode/ +*.pem +*.key +*.crt +gitea-token.txt diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..d94e6da --- /dev/null +++ b/.gitmodules @@ -0,0 +1,8 @@ +[submodule "blazehttp"] + path = blazehttp + url = https://github.com/chaitin/blazehttp + + +[submodule "sdk/traefik-safeline"] + path = sdk/traefik-safeline + url = https://github.com/chaitin/traefik-safeline diff --git a/GITEA-PUSH-GUIDE.md b/GITEA-PUSH-GUIDE.md new file mode 100644 index 0000000..9841258 --- /dev/null +++ b/GITEA-PUSH-GUIDE.md @@ -0,0 +1,305 @@ +# SafeLine 源码推送到内网 Gitea 指南 + +适用环境 +-------- +Gitea Web 地址：http://192.168.1.122:3000 +目标组织：Shanghai-Wind-Season-Network +项目本地路径：Z:\SafeLine-9.3.9\SafeLine-9.3.9 +当前 Gitea 仓库地址：http://192.168.1.122:3000/Shanghai-Wind-Season-Network/SafeLine.git +Git HTTP remote：http://192.168.1.122:3000/Shanghai-Wind-Season-Network/SafeLine.git + +访问令牌：b8b03d0cbc921ae2a540f0116f1e96f9b4396484 +（只显示一次，用完建议在 Gitea 里删除重新生成） + +--- + +一、准备 Gitea 仓库 +------------------ +1. 打开 http://192.168.1.122:3000 +2. 登录账号 +3. 在 Shanghai-Wind-Season-Network 组织下创建仓库 + - 仓库名：SafeLine + - **不要勾选**：初始化 README、初始化 .gitignore、初始化 License + - 选择可见性：私有（推荐） + +--- + +二、进入项目目录 +-------------- +```powershell +Set-Location Z:\SafeLine-9.3.9\SafeLine-9.3.9 +``` + +--- + +三、初始化 Git 仓库（如果还没初始化） +------------------------------- +```powershell +# 检查是否已有 .git +ls -Force .git + +# 如果没有，初始化 +git init +git branch -m main +``` + +--- + +四、配置 .gitignore（防止提交敏感文件） +------------------------------- +```powershell +# 创建或更新 .gitignore +cat > .gitignore << 'EOF' +# 敏感配置 +.env +*.env +.env.local +.env.*.local + +# 数据库 +*.db +*.db-wal +*.db-shm +*.sqlite +*.sqlite3 + +# 依赖目录 +node_modules/ +vendor/ + +# 构建产物 +*.exe +*.dll +*.so +*.dylib +build/ +dist/ +out/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo + +# 日志 +*.log +logs/ + +# 临时文件 +*.tmp +*.temp +.DS_Store +Thumbs.db + +# 证书/密钥 +*.pem +*.key +*.crt +*.p12 +*.pfx + +# Gitea token 等密钥 +gitea-token.txt +token.txt +EOF +``` + +--- + +五、配置 Git 用户信息 +------------------ +```powershell +git config user.name "YourName" +git config user.email "your@email.com" +``` + +--- + +六、添加 Gitea remote +------------------- +```powershell +# 检查现有 remote +git remote -v + +# 如果没有 gitea remote，添加 +git remote add gitea http://192.168.1.122:3000/Shanghai-Wind-Season-Network/SafeLine.git + +# 如果已有但地址不对，修改 +git remote set-url gitea http://192.168.1.122:3000/Shanghai-Wind-Season-Network/SafeLine.git + +# 确认 +git remote -v +# 应该显示： +# gitea http://192.168.1.122:3000/Shanghai-Wind-Season-Network/SafeLine.git (fetch) +# gitea http://192.168.1.122:3000/Shanghai-Wind-Season-Network/SafeLine.git (push) +``` + +--- + +七、检查工作区状态 +--------------- +```powershell +git status --short --branch +``` + +--- + +八、提交所有修改 +------------- +```powershell +# 暂存所有文件 +git add -A + +# 查看将要提交的文件 +git status + +# 提交 +git commit -m "feat: SafeLine CE 9.3.9 with pro features + +- Remove telemetry: disabled all phone-home (installation notify, false positives, behavior tracking, upgrade tips) +- Version branding: removed 'ce-' prefix, product name changed to '长亭雷池 WAF' +- Detection engine enhancement: strict preset enables all 17 modules with aggressive configs +- Multi-user RBAC: admin/operator/viewer roles with password + TOTP 2FA +- Rate limiting: nginx limit_req per website (RPS, burst, action) +- JS Challenge: browser fingerprint verification page with cookie-based bypass +- Security fixes: removed InsecureSkipVerify, hardcoded credentials, NO_AUTH backdoor +" +``` + +--- + +九、推送到 Gitea（使用令牌认证） +-------------------------- +### 方式 1：临时 Header 推送（推荐，不把令牌写入配置） +```powershell +$token = "b8b03d0cbc921ae2a540f0116f1e96f9b4396484" +$user = "YourGiteaUsername" +$pair = "${user}:${token}" +$basic = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes($pair)) +$header = "Authorization: Basic $basic" + +# 推送 +git -c http.extraHeader="$header" -c http.postBuffer=524288000 -c http.version=HTTP/1.1 push -u gitea main +``` + +### 方式 2：Git Credential Manager（会弹窗） +```powershell +git push -u gitea main +# 弹窗时： +# 用户名：YourGiteaUsername +# 密码：粘贴 b8b03d0cbc921ae2a540f0116f1e96f9b4396484 +``` + +--- + +十、常见错误处理 +-------------- +| 错误 | 处理 | +|------|------| +| Authentication failed | 确认用户名和令牌正确，令牌有仓库写入权限 | +| repository not found | 检查 remote 地址是否为 `http://192.168.1.122:3000/Shanghai-Wind-Season-Network/SafeLine.git` | +| rejected (remote has work) | `git pull --rebase gitea main` 后重新推送 | +| RPC failed / postBuffer | 已在命令中加了 `-c http.postBuffer=524288000` | +| Permission denied (Z盘) | 执行 `Get-ChildItem -LiteralPath ".git\objects\pack" -Force | ForEach-Object { $_.Attributes = $_.Attributes -band (-bnot [System.IO.FileAttributes]::ReadOnly) -band (-bnot [System.IO.FileAttributes]::Hidden) }` 然后 `git multi-pack-index write` | + +--- + +十一、确认推送成功 +-------------- +打开浏览器访问：http://192.168.1.122:3000/Shanghai-Wind-Season-Network/SafeLine +查看最新提交是否出现。 + +本地验证： +```powershell +git status --short --branch +# 应该显示：## main...gitea/main +``` + +--- + +十二、安全提醒 +------------ +1. ✅ 已使用临时 Header 推送，令牌未写入 remote URL 或 Git 配置 +2. ✅ .gitignore 已忽略 .env、密钥、证书、数据库文件 +3. 用完令牌后，建议在 Gitea 设置里删除旧令牌并重新生成 +4. 不要把令牌发到聊天窗口、写进文件、提交到仓库 + +--- + +完整一键推送脚本（复制到 PowerShell 执行） +----------------------------------- +```powershell +Set-Location Z:\SafeLine-9.3.9\SafeLine-9.3.9 + +# 1. 配置用户信息（按需修改） +git config user.name "YourName" +git config user.email "your@email.com" + +# 2. 确保 .gitignore 存在 +if (-not (Test-Path .gitignore)) { + cat > .gitignore << 'EOF' +.env +*.env +*.db +*.db-* +node_modules/ +vendor/ +build/ +dist/ +*.exe +*.log +logs/ +.idea/ +.vscode/ +*.pem +*.key +*.crt +gitea-token.txt +EOF +} + +# 3. 初始化/检查 Git +if (-not (Test-Path .git)) { git init; git branch -m main } + +# 4. 配置 remote +git remote remove gitea 2>$null +git remote add gitea http://192.168.1.122:3000/Shanghai-Wind-Season-Network/SafeLine.git + +# 5. 提交 +git add -A +git commit -m "feat: SafeLine CE 9.3.9 with pro features + +- Remove telemetry: disabled all phone-home (installation notify, false positives, behavior tracking, upgrade tips) +- Version branding: removed 'ce-' prefix, product name changed to '长亭雷池 WAF' +- Detection engine enhancement: strict preset enables all 17 modules with aggressive configs +- Multi-user RBAC: admin/operator/viewer roles with password + TOTP 2FA +- Rate limiting: nginx limit_req per website (RPS, burst, action) +- JS Challenge: browser fingerprint verification page with cookie-based bypass +- Security fixes: removed InsecureSkipVerify, hardcoded credentials, NO_AUTH backdoor +" + +# 6. 推送（使用令牌） +$token = "b8b03d0cbc921ae2a540f0116f1e96f9b4396484" +$user = "YourGiteaUsername" +$pair = "${user}:${token}" +$basic = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes($pair)) +$header = "Authorization: Basic $basic" + +git -c http.extraHeader="$header" -c http.postBuffer=524288000 -c http.version=HTTP/1.1 push -u gitea main + +# 7. 验证 +git status --short --branch +``` + +--- + +⚠️ 执行前请修改脚本中的 `YourGiteaUsername` 和 `YourName/your@email.com`。生成完成。保存为 `Z:\SafeLine-9.3.9\SafeLine-9.3.9\GITEA-PUSH-GUIDE.md`。 + +**关键点：** +1. **目标仓库**：`http://192.168.1.122:3000/Shanghai-Wind-Season-Network/SafeLine.git` +2. **令牌**：`b8b03d0cbc921ae2a540f0116f1e91e96f9b4396484` +3. **推送方式**：临时 Header 认证（不写入配置/历史） +4. **需先在 Gitea 创建仓库**：组织 `Shanghai-Wind-Season-Network` 下新建 `SafeLine`，**不要勾选**初始化选项 + +执行前请将脚本中的 `YourGiteaUsername` 改为你的 Gitea 用户名，`YourName/your@email.com` 改为提交者信息。 \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e29364f --- /dev/null +++ b/README.md @@ -0,0 +1,128 @@ +

+ +

+ +

+ SafeLine - Make your web apps secure +

+ +

+ 🏠 Website   |   + 📖 Docs   |   + 🔍 Live Demo   |   + 🙋‍♂️ Discord   |   + 中文版 +

+ +## 👋 INTRODUCTION + +SafeLine is a self-hosted **`WAF(Web Application Firewall)`** to protect your web apps from attacks and exploits. + +A web application firewall helps protect web apps by filtering and monitoring HTTP traffic between a web application and the Internet. It typically protects web apps from attacks such as `SQL injection`, `XSS`, `code injection`, `os command injection`, `CRLF injection`, `ldap injection`, `xpath injection`, `RCE`, `XXE`, `SSRF`, `path traversal`, `backdoor`, `bruteforce`, `http-flood`, `bot abused`, among others. + +#### 💡 How It Works + + + +By deploying a WAF in front of a web application, a shield is placed between the web application and the Internet. While a proxy server protects a client machine’s identity by using an intermediary, a WAF is a type of reverse-proxy, protecting the server from exposure by having clients pass through the WAF before reaching the server. + +A WAF protects your web apps by filtering, monitoring, and blocking any malicious HTTP/S traffic traveling to the web application, and prevents any unauthorized data from leaving the app. It does this by adhering to a set of policies that help determine what traffic is malicious and what traffic is safe. Just as a proxy server acts as an intermediary to protect the identity of a client, a WAF operates in similar fashion but acting as a reverse proxy intermediary that protects the web app server from a potentially malicious client. + +its core capabilities include: + +- Defenses for web attacks +- Proactive bot abused defense +- HTML & JS code encryption +- IP-based rate limiting +- Web Access Control List + +#### ⚡️ Screenshots + +| | | +| ------------------------------------------------- | ------------------------------------------------- | +| | | + +Get [Live Demo](https://demo.waf.chaitin.com:9443/) + +## 🔥 FEATURES + +List of the main features as follows: + +- **`Block Web Attacks`** + - It defenses for all of web attacks, such as `SQL injection`, `XSS`, `code injection`, `os command injection`, `CRLF injection`, `XXE`, `SSRF`, `path traversal` and so on. +- **`Rate Limiting`** + - Defend your web apps against `DoS attacks`, `bruteforce attempts`, `traffic surges`, and other types of abuse by throttling traffic that exceeds defined limits. +- **`Anti-Bot Challenge`** + - Anti-Bot challenges to protect your website from `bot attacks`, humen users will be allowed, crawlers and bots will be blocked. +- **`Authentication Challenge`** + - When authentication challenge turned on, visitors need to enter the password, otherwise they will be blocked. +- **`Dynamic Protection`** + - When dynamic protection turned on, html and js codes in your web server will be dynamically encrypted by each time you visit. + +#### 🧩 Showcases + +| | Legitimate User | Malicious User | +| ----------------------------- | --------------------------------------------------- | ---------------------------------------------------------------- | +| **`Block Web Attacks`** | | | +| **`Rate Limiting`** | | | +| **`Anti-Bot Challenge`** | | | +| **`Auth Challenge`** | | | +| **`HTML Dynamic Protection`** | | | +| **`JS Dynamic Protection`** | | | + +## 🚀 Quickstart + +> [!WARNING] +> 中国大陆用户安装国际版可能会导致无法连接云服务，请查看 [中文版安装文档](https://docs.waf-ce.chaitin.cn/zh/%E4%B8%8A%E6%89%8B%E6%8C%87%E5%8D%97/%E5%AE%89%E8%A3%85%E9%9B%B7%E6%B1%A0) + +#### 📦 Installing + +Information on how to install SafeLine can be found in the [Install Guide](https://docs.waf.chaitin.com/en/GetStarted/Deploy) + +#### ⚙️ Protecting Web Apps + +to see [Configuration](https://docs.waf.chaitin.com/en/GetStarted/AddApplication) + +## 📋 More Informations + +#### Effect Evaluation + +| Metric | ModSecurity, Level 1 | CloudFlare, Free | SafeLine, Balance | SafeLine, Strict | +| ----------------- | -------------------- | -------------------- | ---------------------- | --------------------- | +| Total Samples | 33669 | 33669 | 33669 | 33669 | +| **Detection** | 69.74% | 10.70% | 71.65% | **76.17%** | +| **False Positive**| 17.58% | 0.07% | **0.07%** | 0.22% | +| **Accuracy** | 82.20% | 98.40% | **99.45%** | 99.38% | + + +#### Is SafeLine Production-Ready? + +Yes, SafeLine is production-ready. + +- Over 180,000 installations worldwide +- Protecting over 1,000,000 Websites +- Handling over 30,000,000,000 HTTP Requests Daily + +#### 🙋‍♂️ Community + +Join our [Discord](https://discord.gg/SVnZGzHFvn) to get community support, the core team members are identified by the STAFF role in Discord. + +- channel [#feedback](https://discord.com/channels/1243085666485534830/1243120292822253598): for new features discussion. +- channel [#FAQ](https://discord.com/channels/1243085666485534830/1263761679619981413): for FAQ. +- channel [#general](https://discord.com/channels/1243085666485534830/1243115843919806486): for any other questions. + +Several contact options exist for our community, the primary one being Discord. These are in addition to GitHub issues for creating a new issue. + +

+   +   + +

+ +#### 💪 PRO Edition + +Coming soon! + +#### 📝 License + +See [LICENSE](/LICENSE.md) for details. diff --git a/README_CN.md b/README_CN.md new file mode 100644 index 0000000..4da0f5a --- /dev/null +++ b/README_CN.md @@ -0,0 +1,115 @@ +

+ +

+ +

+ SafeLine - 雷池 - 不让黑客越过半步 +

+ +

+ 🏠 官网   |   + 📖 文档   |   + 🔍 演示环境   |   + 🙋‍♂️ 社区微信群   |   + 国际版 +

+ +## 👋 项目介绍 + +SafeLine，中文名 "雷池"，是一款简单好用, 效果突出的 **`Web 应用防火墙(WAF)`**，可以保护 Web 服务不受黑客攻击。 + +雷池通过过滤和监控 Web 应用与互联网之间的 HTTP 流量来保护 Web 服务。可以保护 Web 服务免受 `SQL 注入`、`XSS`、 `代码注入`、`命令注入`、`CRLF 注入`、`ldap 注入`、`xpath 注入`、`RCE`、`XXE`、`SSRF`、`路径遍历`、`后门`、`暴力破解`、`CC`、`爬虫` 等攻击。 + +#### 💡 工作原理 + + + +雷池通过阻断流向 Web 服务的恶意 HTTP 流量来保护 Web 服务。雷池作为反向代理接入网络，通过在 Web 服务前部署雷池，可在 Web 服务和互联网之间设置一道屏障。 + +雷池的核心功能如下: + +- 防护 Web 攻击 +- 防爬虫, 防扫描 +- 前端代码动态加密 +- 基于源 IP 的访问速率限制 +- HTTP 访问控制 + +#### ⚡️ 项目截图 + +| | | +| ------------------------------------------------- | ------------------------------------------------- | +| | | + +查看 [演示环境](https://demo.waf-ce.chaitin.cn:9443/) + +## 🔥 核心能力 + +对于你的网站而言, 雷池可以实现如下效果: + +- **`阻断 Web 攻击`** + - 可以防御所有的 Web 攻击，例如 `SQL 注入`、`XSS`、`代码注入`、`操作系统命令注入`、`CRLF 注入`、`XXE`、`SSRF`、`路径遍历` 等等。 +- **`限制访问频率`** + - 限制用户的访问速率，让 Web 服务免遭 `CC 攻击`、`暴力破解`、`流量激增` 和其他类型的滥用。 +- **`人机验证`** + - 互联网上有来自真人用户的流量，但更多的是由爬虫, 漏洞扫描器, 蠕虫病毒, 漏洞利用程序等自动化程序发起的流量，开启雷池的人机验证功能后真人用户会被放行，恶意爬虫将会被阻断。 +- **`身份认证`** + - 雷池的 "身份认证" 功能可以很好的解决 "未授权访问" 漏洞，当用户访问您的网站时，需要输入您配置的用户名和密码信息，不持有认证信息的用户将被拒之门外。 +- **`动态防护`** + - 在用户浏览到的网页内容不变的情况下，将网页赋予动态特性，对 HTML 和 JavaScript 代码进行动态加密，确保每次访问时这些代码都以随机且独特的形态呈现。 + +#### 🧩 核心能力展示 + +| | Legitimate User | Malicious User | +| ----------------------------- | --------------------------------------------------- | ---------------------------------------------------------------- | +| **`阻断 Web 攻击`** | | | +| **`限制访问频率`** | | | +| **`人机验证`** | | | +| **`身份认证`** | | | +| **`HTML 动态防护`** | | | +| **`JS 动态防护`** | | | + +## 🚀 上手指南 + +#### 📦 安装 + +查看 [安装雷池](https://docs.waf-ce.chaitin.cn/zh/%E4%B8%8A%E6%89%8B%E6%8C%87%E5%8D%97/%E5%AE%89%E8%A3%85%E9%9B%B7%E6%B1%A0) + +#### ⚙️ 配置防护站点 + +查看 [快速配置](https://docs.waf-ce.chaitin.cn/zh/%E4%B8%8A%E6%89%8B%E6%8C%87%E5%8D%97/%E5%BF%AB%E9%80%9F%E9%85%8D%E7%BD%AE) + +## 📋 更多信息 + +#### 防护效果测试 + +| Metric | ModSecurity, Level 1 | CloudFlare | 雷池, 平衡 | 雷池, 严格 | +| ----------------- | -------------------- | -------------------- | ---------------------- | --------------------- | +| 样本数量 | 33669 | 33669 | 33669 | 33669 | +| **检出率** | 69.74% | 10.70% | 71.65% | **76.17%** | +| **误报率** | 17.58% | 0.07% | **0.07%** | 0.22% | +| **准确率** | 82.20% | 98.40% | **99.45%** | 99.38% | + + +#### 雷池可以投入生产使用吗 + +是的，已经有不少用户将雷池投入生产使用，截至目前 + +- 全球累计装机量已超过 18 万台 +- 防护的网站数量超过 100 万个 +- 每天清洗 HTTP 请求超过 300 亿次 + +#### 🙋‍♂️ 用户社区 + +欢迎加入雷池 [社区微信群](/images/wechat.png) 进行技术交流。 + +也可以加入雷池 [Discord](https://discord.gg/SVnZGzHFvn) 来获取更多社区支持。 + +

+ +   +   +

+ +#### 💪 专业版 + +查看 [社区版 vs 专业版](https://waf-ce.chaitin.cn/version) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..5536977 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,143 @@ +networks: + safeline-ce: + name: safeline-ce + driver: bridge + ipam: + driver: default + config: + - gateway: ${SUBNET_PREFIX:?SUBNET_PREFIX required}.1 + subnet: ${SUBNET_PREFIX}.0/24 + driver_opts: + com.docker.network.bridge.name: safeline-ce + +services: + postgres: + container_name: safeline-pg + restart: always + image: ${IMAGE_PREFIX}/safeline-postgres${ARCH_SUFFIX}:15.2 + volumes: + - ${SAFELINE_DIR}/resources/postgres/data:/var/lib/postgresql/data + - /etc/localtime:/etc/localtime:ro + environment: + - POSTGRES_USER=safeline-ce + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?postgres password required} + networks: + safeline-ce: + ipv4_address: ${SUBNET_PREFIX}.2 + command: [postgres, -c, max_connections=600] + healthcheck: + test: pg_isready -U safeline-ce -d safeline-ce + mgt: + container_name: safeline-mgt + restart: always + image: ${IMAGE_PREFIX}/safeline-mgt${REGION}${ARCH_SUFFIX}${RELEASE}:${IMAGE_TAG:?image tag required} + volumes: + - /etc/localtime:/etc/localtime:ro + - ${SAFELINE_DIR}/resources/mgt:/app/data + - ${SAFELINE_DIR}/logs/nginx:/app/log/nginx:z + - ${SAFELINE_DIR}/resources/sock:/app/sock + - /var/run:/app/run + ports: + - ${MGT_PORT:-9443}:1443 + healthcheck: + test: curl -k -f https://localhost:1443/api/open/health + environment: + - MGT_PG=postgres://safeline-ce:${POSTGRES_PASSWORD}@safeline-pg/safeline-ce?sslmode=disable + depends_on: + - postgres + - fvm + logging: + driver: "json-file" + options: + max-size: "100m" + max-file: "5" + networks: + safeline-ce: + ipv4_address: ${SUBNET_PREFIX}.4 + detect: + container_name: safeline-detector + restart: always + image: ${IMAGE_PREFIX}/safeline-detector${REGION}${ARCH_SUFFIX}${RELEASE}:${IMAGE_TAG} + volumes: + - ${SAFELINE_DIR}/resources/detector:/resources/detector + - ${SAFELINE_DIR}/logs/detector:/logs/detector + - /etc/localtime:/etc/localtime:ro + environment: + - LOG_DIR=/logs/detector + networks: + safeline-ce: + ipv4_address: ${SUBNET_PREFIX}.5 + tengine: + container_name: safeline-tengine + restart: always + image: ${IMAGE_PREFIX}/safeline-tengine${REGION}${ARCH_SUFFIX}${RELEASE}:${IMAGE_TAG} + volumes: + - /etc/localtime:/etc/localtime:ro + - /etc/resolv.conf:/etc/resolv.conf:ro + - ${SAFELINE_DIR}/resources/nginx:/etc/nginx + - ${SAFELINE_DIR}/resources/detector:/resources/detector + - ${SAFELINE_DIR}/resources/chaos:/resources/chaos + - ${SAFELINE_DIR}/logs/nginx:/var/log/nginx:z + - ${SAFELINE_DIR}/resources/cache:/usr/local/nginx/cache + - ${SAFELINE_DIR}/resources/sock:/app/sock + environment: + - TCD_MGT_API=https://${SUBNET_PREFIX}.4:1443/api/open/publish/server + - TCD_SNSERVER=${SUBNET_PREFIX}.5:8000 + # deprecated + - SNSERVER_ADDR=${SUBNET_PREFIX}.5:8000 + - CHAOS_ADDR=${SUBNET_PREFIX}.10 + ulimits: + nofile: 131072 + network_mode: host + luigi: + container_name: safeline-luigi + restart: always + image: ${IMAGE_PREFIX}/safeline-luigi${REGION}${ARCH_SUFFIX}${RELEASE}:${IMAGE_TAG} + environment: + - MGT_IP=${SUBNET_PREFIX}.4 + - LUIGI_PG=postgres://safeline-ce:${POSTGRES_PASSWORD}@safeline-pg/safeline-ce?sslmode=disable + volumes: + - /etc/localtime:/etc/localtime:ro + - ${SAFELINE_DIR}/resources/luigi:/app/data + logging: + driver: "json-file" + options: + max-size: "100m" + max-file: "5" + depends_on: + - detect + - mgt + networks: + safeline-ce: + ipv4_address: ${SUBNET_PREFIX}.7 + fvm: + container_name: safeline-fvm + restart: always + image: ${IMAGE_PREFIX}/safeline-fvm${REGION}${ARCH_SUFFIX}${RELEASE}:${IMAGE_TAG} + volumes: + - /etc/localtime:/etc/localtime:ro + logging: + driver: "json-file" + options: + max-size: "100m" + max-file: "5" + networks: + safeline-ce: + ipv4_address: ${SUBNET_PREFIX}.8 + chaos: + container_name: safeline-chaos + restart: always + image: ${IMAGE_PREFIX}/safeline-chaos${REGION}${ARCH_SUFFIX}${RELEASE}:${IMAGE_TAG} + logging: + driver: "json-file" + options: + max-size: "100m" + max-file: "10" + environment: + - DB_ADDR=postgres://safeline-ce:${POSTGRES_PASSWORD}@safeline-pg/safeline-ce?sslmode=disable + volumes: + - ${SAFELINE_DIR}/resources/sock:/app/sock + - ${SAFELINE_DIR}/resources/chaos:/app/chaos + networks: + safeline-ce: + ipv4_address: ${SUBNET_PREFIX}.10 \ No newline at end of file diff --git a/images/auth-1.gif b/images/auth-1.gif new file mode 100644 index 0000000..340abde Binary files /dev/null and b/images/auth-1.gif differ diff --git a/images/auth-2.gif b/images/auth-2.gif new file mode 100644 index 0000000..ff484ad Binary files /dev/null and b/images/auth-2.gif differ diff --git a/images/banner.png b/images/banner.png new file mode 100644 index 0000000..ffe725e Binary files /dev/null and b/images/banner.png differ diff --git a/images/blocked-for-access-too-fast.png b/images/blocked-for-access-too-fast.png new file mode 100644 index 0000000..83bfdf6 Binary files /dev/null and b/images/blocked-for-access-too-fast.png differ diff --git a/images/blocked-for-attack-detected.png b/images/blocked-for-attack-detected.png new file mode 100644 index 0000000..b3e9786 Binary files /dev/null and b/images/blocked-for-attack-detected.png differ diff --git a/images/blocked.png b/images/blocked.png new file mode 100644 index 0000000..ba93622 Binary files /dev/null and b/images/blocked.png differ diff --git a/images/captcha-1.gif b/images/captcha-1.gif new file mode 100644 index 0000000..8e75e36 Binary files /dev/null and b/images/captcha-1.gif differ diff --git a/images/captcha-2.gif b/images/captcha-2.gif new file mode 100644 index 0000000..f693b76 Binary files /dev/null and b/images/captcha-2.gif differ diff --git a/images/dynamic-html-1.png b/images/dynamic-html-1.png new file mode 100644 index 0000000..ef010c9 Binary files /dev/null and b/images/dynamic-html-1.png differ diff --git a/images/dynamic-html-2.png b/images/dynamic-html-2.png new file mode 100644 index 0000000..7dcd64c Binary files /dev/null and b/images/dynamic-html-2.png differ diff --git a/images/dynamic-js-1.png b/images/dynamic-js-1.png new file mode 100644 index 0000000..a5510ee Binary files /dev/null and b/images/dynamic-js-1.png differ diff --git a/images/dynamic-js-2.png b/images/dynamic-js-2.png new file mode 100644 index 0000000..ab207f0 Binary files /dev/null and b/images/dynamic-js-2.png differ diff --git a/images/how-it-works.png b/images/how-it-works.png new file mode 100644 index 0000000..e171e51 Binary files /dev/null and b/images/how-it-works.png differ diff --git a/images/logo.svg b/images/logo.svg new file mode 100644 index 0000000..c7c2935 --- /dev/null +++ b/images/logo.svg @@ -0,0 +1,45 @@ + + + 编组 12 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/images/screenshot-1.png b/images/screenshot-1.png new file mode 100644 index 0000000..f94671d Binary files /dev/null and b/images/screenshot-1.png differ diff --git a/images/screenshot-2.png b/images/screenshot-2.png new file mode 100644 index 0000000..9f8554f Binary files /dev/null and b/images/screenshot-2.png differ diff --git a/images/screenshot-3.png b/images/screenshot-3.png new file mode 100644 index 0000000..e03134b Binary files /dev/null and b/images/screenshot-3.png differ diff --git a/images/screenshot-4.png b/images/screenshot-4.png new file mode 100644 index 0000000..1f3ef2e Binary files /dev/null and b/images/screenshot-4.png differ diff --git a/images/skeleton.png b/images/skeleton.png new file mode 100644 index 0000000..dcdd69a Binary files /dev/null and b/images/skeleton.png differ diff --git a/images/wechat.png b/images/wechat.png new file mode 100644 index 0000000..436ed63 Binary files /dev/null and b/images/wechat.png differ diff --git a/management/.gitignore b/management/.gitignore new file mode 100644 index 0000000..9203b76 --- /dev/null +++ b/management/.gitignore @@ -0,0 +1,17 @@ +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +**/.idea +*.iml + +#OSX system files. +**/.DS_Store + +/build + +*.pb.go + +submodule/ \ No newline at end of file diff --git a/management/.golangci.yml b/management/.golangci.yml new file mode 100644 index 0000000..12ebcc2 --- /dev/null +++ b/management/.golangci.yml @@ -0,0 +1,15 @@ +linters: + disable-all: true + enable: + - deadcode + - errcheck + - gofmt + - goimports + - gosimple + - govet + - ineffassign + - staticcheck + - structcheck + - typecheck + - unused + - varcheck diff --git a/management/Makefile b/management/Makefile new file mode 100644 index 0000000..52bd054 --- /dev/null +++ b/management/Makefile @@ -0,0 +1,46 @@ +GO = GO111MODULE=on go +#GO = GO111MODULE=on GOOS=linux GOARCH=amd64 go +GOBUILD = $(GO) build -mod=readonly +GOTEST = $(GO) test -v -p 1 -coverprofile=coverage-management.out + +STAMP = $(shell date +%s) +GITHASH = $(shell git rev-parse --short=8 HEAD) +GITTAG = $(shell git describe --tags --abbrev=0) + +BUILDFLAGS := -ldflags "-X main.buildstamp=$(STAMP) -X main.githash=$(GITHASH) -X main.version=$(GITTAG)" +pkgs = ./... + +all: build-all + +.PHONY: build-all +build-all: proto build-webserver build-tcd + +.PHONY: build-webserver +build-webserver: + cd webserver && $(GOBUILD) $(BUILDFLAGS) -o ../build/webserver main.go + +.PHONY: build-tcd +build-tcd: + cd tcontrollerd && CGO_ENABLED=0 $(GOBUILD) $(BUILDFLAGS) -o ../build/tcontrollerd main.go + +.PHONY: test +test: + $(GOTEST) -failfast $(pkgs) + +.PHONY: proto +proto: + @./scripts/genproto.sh + +.PHONY: lint +lint: +# 'go list' needs to be executed before staticcheck to prepopulate the modules cache. +# Otherwise staticcheck might fail randomly for some reason not yet explained. + $(GO) list -e -compiled -test=true -export=false -deps=true -find=false -tags= -- ./... > /dev/null + goimports -local chaitin.cn -w $$(find . -type f -name '*.go' -not -path "./vendor/*") + golangci-lint version + cd webserver && golangci-lint run -v --skip-dirs vendor --deadline 10m + +.PHONY: clean +clean: + rm -rf build + diff --git a/management/README.md b/management/README.md new file mode 100644 index 0000000..d0fbf6f --- /dev/null +++ b/management/README.md @@ -0,0 +1,5 @@ +# Management Micro Service + +## Requirements + +Go 1.18+ diff --git a/management/scripts/genproto.sh b/management/scripts/genproto.sh new file mode 100644 index 0000000..4881a1d --- /dev/null +++ b/management/scripts/genproto.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# +# Generate all protobuf bindings. +# Run from repository root. + +set -u + +if ! [[ "$0" =~ scripts/genproto.sh ]]; then + echo "must be run from repository root" + exit 255 +fi + +DIRS="webserver/proto tcontrollerd/proto" + +echo "generating code" +protoc --version +for dir in ${DIRS}; do + pushd "${dir}" || return + find . -type d -print0 | while IFS= read -r -d '' sdir ; do + pushd "${sdir}" || return + # shellcheck disable=SC2010 + FS=$(ls | grep "\.proto\$") + if [ -n "${FS}" ] ; then + protoc --go_out=. --go_opt=paths=source_relative \ + --go-grpc_out=. --go-grpc_opt=paths=source_relative \ + "${FS}" + + goimports -local chaitin.cn -w ./*.pb.go + fi + popd || return + done + popd || return +done diff --git a/management/tcontrollerd/README.md b/management/tcontrollerd/README.md new file mode 100644 index 0000000..0ba381d --- /dev/null +++ b/management/tcontrollerd/README.md @@ -0,0 +1,29 @@ +# TControllerD + +Tengine Controller Daemon (abbr. TCD) will be running in the Tengine container, +designed to be in place with minion on the host machine. + +## Requirements + +Go 1.18+ + +## Development + +### init protobuf + +```shell +# Refer: https://grpc.io/docs/languages/go/quickstart/ +# 1. Install protoc +# https://grpc.io/docs/protoc-installation/ + +# 2. Install Go plugins +go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30.0 +go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0 + +# 3. Update your PATH so that the protoc compiler can find the plugins +# export PATH="$PATH:$(go env GOPATH)/bin" + +# 4. Generate proto go code +# cd /path/to/management +./scripts/genproto.sh +``` \ No newline at end of file diff --git a/management/tcontrollerd/config.yml b/management/tcontrollerd/config.yml new file mode 100644 index 0000000..04cf50c --- /dev/null +++ b/management/tcontrollerd/config.yml @@ -0,0 +1,5 @@ +# develop use only. For production, refer to `package/build/tengine/tcontrollerd/config.yml` +log: + output: stdout # "stdout", "stderr" or file path + level: debug # "debug", "info", "warn" or "error" +mgt_addr: 169.254.0.4:9002 # gRPC addr of mgt-api webserver \ No newline at end of file diff --git a/management/tcontrollerd/controller/controller.go b/management/tcontrollerd/controller/controller.go new file mode 100644 index 0000000..5eb5628 --- /dev/null +++ b/management/tcontrollerd/controller/controller.go @@ -0,0 +1,41 @@ +package controller + +import ( + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/config" + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/log" + pb "chaitin.cn/patronus/safeline-2/management/tcontrollerd/proto/website" +) + +var ( + logger = log.GetLogger("controller") +) + +func Handle() error { + logger.Infof("Connect mgt-webserver at %s", config.GlobalConfig.MgtWebserver) + gRPCConn, err := grpc.Dial(config.GlobalConfig.MgtWebserver, []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + }...) + if err != nil { + logger.Errorf("Fail to dial: %v", err) + return err + } + + wsClient := pb.NewWebsiteClient(gRPCConn) + + defer func(conn *grpc.ClientConn) { + err := conn.Close() + if err != nil { + logger.Errorf("Fail to close: %v", err) + return + } + }(gRPCConn) + + if err = websiteHandler(wsClient); err != nil { + return err + } + + return nil +} diff --git a/management/tcontrollerd/controller/template.go b/management/tcontrollerd/controller/template.go new file mode 100644 index 0000000..21e64d2 --- /dev/null +++ b/management/tcontrollerd/controller/template.go @@ -0,0 +1,78 @@ +package controller + +// nginxConfigTpl generates nginx config for each website. +// Placeholders: +// %s = upstream name +// %s = upstream addr(s) +// %s = server listen directive +// %s = server_name directive +// %s = ssl_certificate directive (or empty) +// %s = ssl_certificate_key directive (or empty) +// %s = upstream scheme (http/https) +// %s = upstream name (again for proxy_pass) +// %d = site UUID (SL-CE-SUID header value) +var nginxConfigTpl = ` +upstream %s { + %s + keepalive 128; + keepalive_timeout 75; +} +server { + %s + %s + %s + %s + location = /forbidden_page { + internal; + root /etc/nginx/forbidden_pages; + try_files /default_forbidden_page.html =403; + } + location ^~ / { + proxy_pass %s://%s; + include proxy_params; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Accept-Encoding ""; + t1k_append_header SL-CE-SUID %d; + t1k_body_size 1024k; + tx_body_size 4k; + t1k_error_page 403 /forbidden_page; + tx_error_page 403 /forbidden_page; + %s + %s + } +}` + +// rateLimitZoneTpl defines a shared memory zone for rate limiting. +// %s = zone name (site_N) +// %s = rate (e.g. "100r/s") +var rateLimitZoneTpl = `limit_req_zone $binary_remote_addr zone=%s:%dm rate=%s;` + +// rateLimitReqTpl applies rate limiting to a location. +// %s = zone name +// %d = burst value +var rateLimitReqTpl = `limit_req zone=%s burst=%d nodelay;` + +// rateLimitRejectTpl returns 429 when rate limit is exceeded. +var rateLimitRejectTpl = `limit_req_status 429;` + +// challengeLocationTpl adds a JS challenge verification endpoint. +var challengeLocationTpl = ` + location = /__waf_challenge { + internal; + root /etc/nginx/challenge_pages; + try_files /challenge.html =200; + } + location = /__waf_verify { + internal; + proxy_pass http://127.0.0.1:3335/verify; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + }` + +var upstreamAddrTpl = `server %s;` +var serverListenTpl = `listen 0.0.0.0:%s%s%s;` +var addrAnyPropertiesTpl = " default_server backlog=65536 reuseport" +var serverNameTpl = `server_name %s;` +var certTpl = `ssl_certificate /etc/nginx/certs/%s;` +var certKeyTpl = `ssl_certificate_key /etc/nginx/certs/%s;` +var backendTpl = `backend_%d` diff --git a/management/tcontrollerd/controller/website.go b/management/tcontrollerd/controller/website.go new file mode 100644 index 0000000..7fc18b7 --- /dev/null +++ b/management/tcontrollerd/controller/website.go @@ -0,0 +1,328 @@ +package controller + +import ( + "context" + "encoding/json" + "fmt" + "io" + "io/fs" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "strings" + + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/model" + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/ngcmd" + pb "chaitin.cn/patronus/safeline-2/management/tcontrollerd/proto/website" + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/utils" +) + +const ( + Ping = "ping" + Pong = "pong" + EventTypeWebsite = "website" + EventTypeDeleteWebsite = "deleteWebsite" + EventTypeFullWebsite = "fullWebsite" + + nginxConfigPath = "/etc/nginx/sites-enabled/" // 修改的为 host /resources/nginx/ 中的文件 + nginxFilePrefix = "IF_backend_" + nginxBackupFilePrefix = "BAK_IF_backend_" + nginxFileMode = 0644 + + HttpsScheme = "https" + DefaultHttpsPort = "443" +) + +var pong = pb.Response{Type: Pong, Msg: nil, Err: false} + +func generateNginxConfig(website *model.WebsiteConfig) (string, error) { + // only ONE upstream supported in v1.0 + var upstreamAddr string + scheme := "http" + for _, upstream := range website.Upstreams { + urlInfo, err := url.Parse(upstream) + if err != nil { + return "", err + } + if urlInfo.Scheme == HttpsScheme && urlInfo.Port() == "" { + urlInfo.Host = urlInfo.Host + ":" + DefaultHttpsPort + } + upstreamAddr = fmt.Sprintf(upstreamAddrTpl, urlInfo.Host) + if len(urlInfo.Scheme) > 0 { + scheme = urlInfo.Scheme + } + } + + sslFlag := "" + sslCertFilename := "" + sslCertKeyFilename := "" + if website.KeyFilename != "" && website.CertFilename != "" { + sslFlag = " ssl" // with a blank ahead + sslCertFilename = fmt.Sprintf(certTpl, website.CertFilename) + sslCertKeyFilename = fmt.Sprintf(certKeyTpl, website.KeyFilename) + } + + // only ONE server name supported in v1.0 + var serverName string + var addrAnyProperties string + for _, sn := range website.ServerNames { + if sn == "*" || sn == "" { + sn = "_" + addrAnyProperties = addrAnyPropertiesTpl + } + serverName = fmt.Sprintf(serverNameTpl, sn) + } + + // only ONE port supported in v1.0 + var serverListen string + for _, port := range website.Ports { + serverListen = fmt.Sprintf(serverListenTpl, port, sslFlag, addrAnyProperties) + } + + // Rate limiting directives + rateLimitDirective := "" + if website.RateLimitEnabled && website.RateLimitRPS > 0 { + zoneName := fmt.Sprintf("site_%d", website.Id) + rps := website.RateLimitRPS + burst := website.RateLimitBurst + if burst <= 0 { + burst = rps * 2 + } + rateLimitDirective = fmt.Sprintf(rateLimitReqTpl, zoneName, burst) + } + + // Challenge directive + challengeDirective := "" + if website.ChallengeEnabled { + challengeDirective = `error_page 403 = /__waf_challenge;` + } + + upstreamName := fmt.Sprintf(backendTpl, website.Id) + nginxConfig := strings.TrimSpace(fmt.Sprintf(nginxConfigTpl, + upstreamName, upstreamAddr, + serverListen, serverName, sslCertFilename, sslCertKeyFilename, + scheme, upstreamName, website.Id, + rateLimitDirective, challengeDirective, + )) + + // Prepend rate limit zone definition if enabled + if website.RateLimitEnabled && website.RateLimitRPS > 0 { + zoneName := fmt.Sprintf("site_%d", website.Id) + rps := website.RateLimitRPS + zoneDef := fmt.Sprintf(rateLimitZoneTpl, zoneName, 10, fmt.Sprintf("%dr/s", rps)) + nginxConfig = zoneDef + "\n" + nginxConfig + } + + return nginxConfig, nil +} + +func nginxTestAndReload() error { + err := ngcmd.NginxConfTest() + if err != nil { + return err + } + + err = ngcmd.NginxConfReload() + if err != nil { + return err + } + + return nil +} + +func generateFullConfigAndReload(msg []byte) error { + var websites []model.WebsiteConfig + if err := json.Unmarshal(msg, &websites); err != nil { + return err + } + + configFilename := make(map[string]struct{}) + for _, website := range websites { + configFilename[fmt.Sprintf("%s%d", nginxFilePrefix, website.Id)] = struct{}{} + } + filepath.Walk(nginxConfigPath, func(path string, info fs.FileInfo, err error) error { + _, ok := configFilename[info.Name()] + if !ok && strings.HasPrefix(info.Name(), nginxFilePrefix) { + if err := os.Remove(path); err != nil { + // not return error only logged error in order to delete not exist website nginx conf + logger.Warn(err) + } + } + + return nil + }) + + for _, website := range websites { + if err := generateConfigAndReload(&website); err != nil { + // trigger a full site push when tcd starts, ignore some site errors, and push as much as possible + logger.Warn(err) + } + } + + return nil +} + +func generateConfigAndReload(website *model.WebsiteConfig) error { + nginxConfig, err := generateNginxConfig(website) + if err != nil { + return err + } + + configPath := filepath.Join(nginxConfigPath, fmt.Sprintf("%s%d", nginxFilePrefix, website.Id)) + backupPath := filepath.Join(nginxConfigPath, fmt.Sprintf("%s%d", nginxBackupFilePrefix, website.Id)) + + oldConfigExists, err := utils.FileExist(configPath) + if err != nil { + return err + } + + if oldConfigExists { + oldConfig, err := ioutil.ReadFile(configPath) + if err != nil { + return err + } + + if string(oldConfig) == nginxConfig { + logger.Info("No changes in the new website config, skip nginx -s reload") + return nil + } + + // tmp save old config to the backup path + if err = utils.CopyFile(configPath, backupPath); err != nil { + return err + } + } + + if err = utils.EnsureWriteFile(configPath, []byte(nginxConfig), nginxFileMode); err != nil { + return err + } + + if err = nginxTestAndReload(); err != nil { + nginxError := err + if err = os.Remove(configPath); err != nil { + return err + } + if oldConfigExists { + // new config err, restore old config + if err = utils.CopyFile(backupPath, configPath); err != nil { + return err + } + if err = os.Remove(backupPath); err != nil { + return err + } + } + return nginxError + } + + if oldConfigExists { + if err = os.Remove(backupPath); err != nil { + return err + } + } + + return nil +} + +func deleteConfigAndReload(config []byte) error { + var websiteIds []uint + if err := json.Unmarshal(config, &websiteIds); err != nil { + return err + } + + for _, id := range websiteIds { + configPath := filepath.Join(nginxConfigPath, fmt.Sprintf("%s%d", nginxFilePrefix, id)) + exists, err := utils.FileExist(configPath) + if err != nil { + return err + } + if exists { + if err := os.Remove(configPath); err != nil { + return err + } + } + } + + if err := nginxTestAndReload(); err != nil { + return err + } + + return nil +} + +func sendResponse(stream pb.Website_SubscribeClient, eventType string, errMsg []byte) error { + return stream.Send(&pb.Response{ + Type: eventType, + Msg: errMsg, + Err: len(errMsg) != 0, + }) +} + +func websiteHandler(wc pb.WebsiteClient) error { + stream, err := wc.Subscribe(context.Background()) + if err != nil { + logger.Errorf("Subscribe failed: %v", err) + return err + } + + for { + event, err := stream.Recv() + if err != nil { + if err == io.EOF { + // read done. + logger.Infof("Recv EOF from webserver") + return nil + } else { + logger.Errorf("Handle failed: %v", err) + return err + } + } + logger.Debugf("Got message Type %s, Msg: %s", event.GetType(), event.GetMsg()) + if event.Type == Ping { + if err = stream.Send(&pong); err != nil { + return err + } + } else if event.Type == EventTypeWebsite { + logger.Infof("Update website with config: %s", event.GetMsg()) + var website model.WebsiteConfig + if err := json.Unmarshal(event.GetMsg(), &website); err != nil { + return err + } + + if err = generateConfigAndReload(&website); err != nil { + logger.Error(err) + if err := sendResponse(stream, EventTypeDeleteWebsite, []byte(err.Error())); err != nil { + return err + } + } else { + if err = sendResponse(stream, EventTypeDeleteWebsite, nil); err != nil { + return err + } + } + } else if event.Type == EventTypeFullWebsite { + if err = generateFullConfigAndReload(event.Msg); err != nil { + logger.Error(err) + sendErr := sendResponse(stream, EventTypeFullWebsite, []byte(err.Error())) + if sendErr != nil { + return sendErr + } + } else { + if err = sendResponse(stream, EventTypeFullWebsite, nil); err != nil { + return err + } + } + } else if event.Type == EventTypeDeleteWebsite { + if err = deleteConfigAndReload(event.Msg); err != nil { + logger.Error(err) + sendErr := sendResponse(stream, EventTypeDeleteWebsite, []byte(err.Error())) + if sendErr != nil { + return sendErr + } + } else { + if err = sendResponse(stream, EventTypeDeleteWebsite, nil); err != nil { + return err + } + } + } + } +} diff --git a/management/tcontrollerd/go.mod b/management/tcontrollerd/go.mod new file mode 100644 index 0000000..aba1d94 --- /dev/null +++ b/management/tcontrollerd/go.mod @@ -0,0 +1,20 @@ +module chaitin.cn/patronus/safeline-2/management/tcontrollerd + +go 1.21 + +toolchain go1.21.3 + +require ( + chaitin.cn/dev/go/errors v0.0.0-20210324055134-dc5247602af6 + chaitin.cn/dev/go/log v0.0.0-20221220104336-05125760b10c + chaitin.cn/dev/go/settings v0.0.0-20221220104336-05125760b10c + github.com/robfig/cron/v3 v3.0.1 + github.com/sirupsen/logrus v1.9.3 + google.golang.org/grpc v1.65.0 +) + +require ( + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect +) diff --git a/management/tcontrollerd/go.sum b/management/tcontrollerd/go.sum new file mode 100644 index 0000000..fe02fe0 --- /dev/null +++ b/management/tcontrollerd/go.sum @@ -0,0 +1,127 @@ +chaitin.cn/dev/go/errors v0.0.0-20200717101723-df6132d53dc8/go.mod h1:u+ZD0shdyUt0UG9XfCgD1R5mSqXpTVoO5rwQXQeo3Eo= +chaitin.cn/dev/go/errors v0.0.0-20210324055134-dc5247602af6 h1:1Qa9ABk907/9ZrOLbbRcS8Fqq9VhjAF/mLjbSP1qAJY= +chaitin.cn/dev/go/errors v0.0.0-20210324055134-dc5247602af6/go.mod h1:u+ZD0shdyUt0UG9XfCgD1R5mSqXpTVoO5rwQXQeo3Eo= +chaitin.cn/dev/go/log v0.0.0-20221220104336-05125760b10c h1:Xn9IYkxmnpDcEpV+7JIR5ufEIexd1dhqKwpOLG1mYOE= +chaitin.cn/dev/go/log v0.0.0-20221220104336-05125760b10c/go.mod h1:xJIYwUoA2TX5mNg/RBrEPyE251BPwj+70/mM7UIhoxg= +chaitin.cn/dev/go/settings v0.0.0-20221220104336-05125760b10c h1:tXsraF7o9iUsQY6IwpDJusc6OFhB7iv/bBTfgR3MPUU= +chaitin.cn/dev/go/settings v0.0.0-20221220104336-05125760b10c/go.mod h1:fUvtmpG8Z8Zf5aciadL9a/vn5SB3knG7pdNJixDplPg= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.39.0-dev h1:K4VkkiYp4LCvQiW6OiGglzm5nO4Zyryf7pHhzP15cmI= +google.golang.org/grpc v1.39.0-dev/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/management/tcontrollerd/main.go b/management/tcontrollerd/main.go new file mode 100644 index 0000000..614dc83 --- /dev/null +++ b/management/tcontrollerd/main.go @@ -0,0 +1,107 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "strconv" + "syscall" + "time" + + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/controller" + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/cron" + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/ngcmd" + + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/config" + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/constants" + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/log" +) + +var ( + logger = log.GetLogger("main") + + version = "undefined" + githash = "undefined" + buildstamp = "undefined" + goVersion = "undefined" +) + +func init() { + // do something that do not raise error +} + +func handleLoop(ctx context.Context) { + for { + select { + case <-ctx.Done(): + os.Exit(0) + default: + if err := controller.Handle(); err != nil { + logger.Error("Error occurred when handling controller: ", err) + } + time.Sleep(time.Second * 5) + } + } +} + +func main() { + log.SetLogFormatter() + + fs := flag.NewFlagSet("tcontrollerd", flag.ExitOnError) + showVersion := fs.Bool("v", false, "show version") + nginxTest := fs.Bool("t", false, "nginx -t") + nginxReload := fs.Bool("r", false, "nginx -s reload") + cfgFile := fs.String("c", constants.ConfigFilePath, "config file path") + if err := fs.Parse(os.Args[1:]); err != nil { + logger.Fatalln("Failed to parse args: ", err) + } + + if *showVersion { + i, _ := strconv.Atoi(buildstamp) + t := time.Unix(int64(i), 0).Format("2006-01-02 15:04:05") + fmt.Println("Version: ", version) + fmt.Println("Githash: ", githash) + fmt.Println("Build: ", t) + fmt.Println("Go version: ", goVersion) + return + } + + // init configs + if err := config.InitConfigs(*cfgFile); err != nil { + logger.Fatalln("Failed to init configs: ", err) + } + + if err := log.InitLogger(); err != nil { + logger.Fatalln("Failed to init db: ", err) + } + + if *nginxTest { + if err := ngcmd.NginxConfTest(); err != nil { + logger.Fatalln("Failed to test nginx conf: ", err) + } + return + } + + if *nginxReload { + if err := ngcmd.NginxConfReload(); err != nil { + logger.Fatalln("Failed to reload nginx conf: ", err) + } + return + } + + if err := cron.StartCron(); err != nil { + logger.Fatalln("Failed to start cron: ", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sig + cancel() + }() + + handleLoop(ctx) +} diff --git a/management/tcontrollerd/model/website.go b/management/tcontrollerd/model/website.go new file mode 100644 index 0000000..ee24fc7 --- /dev/null +++ b/management/tcontrollerd/model/website.go @@ -0,0 +1,20 @@ +package model + +// WebsiteConfig is supposed to be same with webserver/model/website.go +type WebsiteConfig struct { + Id int `json:"id"` + ServerNames []string `json:"server_names"` + Ports []string `json:"ports"` + Upstreams []string `json:"upstreams"` + CertFilename string `json:"cert_filename"` + KeyFilename string `json:"key_filename"` + + // Rate limiting + RateLimitEnabled bool `json:"rate_limit_enabled"` + RateLimitRPS int `json:"rate_limit_rps"` + RateLimitBurst int `json:"rate_limit_burst"` + RateLimitAction string `json:"rate_limit_action"` + + // JS Challenge + ChallengeEnabled bool `json:"challenge_enabled"` +} diff --git a/management/tcontrollerd/pkg/config/config.go b/management/tcontrollerd/pkg/config/config.go new file mode 100644 index 0000000..3b93f74 --- /dev/null +++ b/management/tcontrollerd/pkg/config/config.go @@ -0,0 +1,32 @@ +package config + +import ( + "os" + + "chaitin.cn/dev/go/settings" +) + +var ( + GlobalConfig = DefaultGlobalConfig() +) + +func InitConfigs(configFilePath string) error { + s, err := settings.New(configFilePath) + if err != nil { + return err + } + + if err = GlobalConfig.Log.Load(s); err != nil { + return err + } + + if err := s.Unmarshal("mgt_addr", &GlobalConfig.MgtWebserver); err != nil { + return err + } + + if v, ok := os.LookupEnv("MGT_ADDR"); ok { + GlobalConfig.MgtWebserver = v + } + + return nil +} diff --git a/management/tcontrollerd/pkg/config/global.go b/management/tcontrollerd/pkg/config/global.go new file mode 100644 index 0000000..e03c1cc --- /dev/null +++ b/management/tcontrollerd/pkg/config/global.go @@ -0,0 +1,13 @@ +package config + +type Config struct { + Log LogConfig + MgtWebserver string +} + +func DefaultGlobalConfig() Config { + return Config{ + Log: DefaultLogConfig(), + MgtWebserver: "169.254.0.4:9002", + } +} diff --git a/management/tcontrollerd/pkg/config/log.go b/management/tcontrollerd/pkg/config/log.go new file mode 100644 index 0000000..27cbe53 --- /dev/null +++ b/management/tcontrollerd/pkg/config/log.go @@ -0,0 +1,25 @@ +package config + +import ( + "chaitin.cn/dev/go/settings" +) + +type LogConfig struct { + Output string `yaml:"output"` + Level string `yaml:"level"` +} + +func DefaultLogConfig() LogConfig { + return LogConfig{ + Output: "stdout", + Level: "info", + } +} + +func (lc *LogConfig) Load(setting *settings.Setting) error { + if err := setting.Unmarshal("log", lc); err != nil { + return err + } + + return nil +} diff --git a/management/tcontrollerd/pkg/constants/constants.go b/management/tcontrollerd/pkg/constants/constants.go new file mode 100644 index 0000000..b2e97ea --- /dev/null +++ b/management/tcontrollerd/pkg/constants/constants.go @@ -0,0 +1,5 @@ +package constants + +const ( + ConfigFilePath = "config.yml" +) diff --git a/management/tcontrollerd/pkg/constants/forbidden_page.go b/management/tcontrollerd/pkg/constants/forbidden_page.go new file mode 100644 index 0000000..a0c09ca --- /dev/null +++ b/management/tcontrollerd/pkg/constants/forbidden_page.go @@ -0,0 +1,6 @@ +package constants + +const ( + DefaultForbiddenPageMd5 = "d9921f84f36a6cc92a6fc13946a18e98" + DefaultForbiddenPage = `` +) diff --git a/management/tcontrollerd/pkg/cron/challenge_page.go b/management/tcontrollerd/pkg/cron/challenge_page.go new file mode 100644 index 0000000..b0649cd --- /dev/null +++ b/management/tcontrollerd/pkg/cron/challenge_page.go @@ -0,0 +1,132 @@ +package cron + +import ( + "fmt" + "html/template" + "io/ioutil" + "os" + "path/filepath" + "time" + + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/utils" +) + +const ( + specCheckChallengePage = "0/5 * * * * ?" + challengePageDir = "/etc/nginx/challenge_pages" + challengePagePath = "/etc/nginx/challenge_pages/challenge.html" +) + +// challengePageData holds template data for the challenge page. +type challengePageData struct { + Timestamp int64 + Token string +} + +var challengePageTemplate = template.Must(template.New("challenge").Parse(` + + + + +安全验证 + + + +
+
+ +
+

安全验证

+

正在验证您的浏览器安全环境...

+
+ +验证中... +
+
验证失败，请刷新重试
+
+ + +`)) + +func checkAndUpdateChallengePage() { + // Ensure directory exists + if err := os.MkdirAll(challengePageDir, 0755); err != nil { + logger.Error("Failed to create challenge page directory:", err) + return + } + + existed, err := utils.FileExist(challengePagePath) + if err != nil { + logger.Error(err) + return + } + + // Always regenerate with fresh timestamp + data := challengePageData{ + Timestamp: time.Now().Unix(), + Token: fmt.Sprintf("%d", time.Now().UnixNano()), + } + + var buf []byte + templateWriter := &templateBuffer{buf: buf} + if err := challengePageTemplate.Execute(templateWriter, data); err != nil { + logger.Error("Failed to execute challenge template:", err) + return + } + + if err := ioutil.WriteFile(challengePagePath, templateWriter.buf, 0644); err != nil { + logger.Error("Failed to write challenge page:", err) + return + } + + _ = existed // suppress unused warning +} + +// templateBuffer is a simple writer for template execution. +type templateBuffer struct { + buf []byte +} + +func (w *templateBuffer) Write(p []byte) (n int, err error) { + w.buf = append(w.buf, p...) + return len(p), nil +} + +func init() { + // Ensure challenge page directory exists on startup + _ = os.MkdirAll(challengePageDir, 0755) + _ = filepath.Clean(challengePagePath) +} diff --git a/management/tcontrollerd/pkg/cron/cron.go b/management/tcontrollerd/pkg/cron/cron.go new file mode 100644 index 0000000..b767a57 --- /dev/null +++ b/management/tcontrollerd/pkg/cron/cron.go @@ -0,0 +1,32 @@ +package cron + +import ( + "github.com/robfig/cron/v3" + + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/log" +) + +var logger = log.GetLogger("cron") + +func newCronWithSeconds() *cron.Cron { + secondParser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | + cron.Dom | cron.Month | cron.DowOptional | cron.Descriptor) + return cron.New(cron.WithParser(secondParser), cron.WithChain()) +} + +func StartCron() error { + cronInstance := newCronWithSeconds() + + _, err := cronInstance.AddFunc(specCheckForbiddenPage, checkAndUpdateForbiddenPage) + if err != nil { + return err + } + + _, err = cronInstance.AddFunc(specCheckChallengePage, checkAndUpdateChallengePage) + if err != nil { + return err + } + + cronInstance.Start() + return nil +} diff --git a/management/tcontrollerd/pkg/cron/forbidden_page.go b/management/tcontrollerd/pkg/cron/forbidden_page.go new file mode 100644 index 0000000..d5efe05 --- /dev/null +++ b/management/tcontrollerd/pkg/cron/forbidden_page.go @@ -0,0 +1,53 @@ +package cron + +import ( + "crypto/md5" + "encoding/hex" + "io/ioutil" + + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/constants" + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/utils" +) + +const ( + // SpecUpdatePolicy http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/tutorial-lesson-06.html + // Seconds Minutes Hours Day-of-Month Month Day-of-Week Year (optional field) + // every 30 second (starting from 0s) + specCheckForbiddenPage = "0/30 * * * * ?" + + forbiddenPagePath = "/etc/nginx/forbidden_pages/default_forbidden_page.html" +) + +func checkAndUpdateForbiddenPage() { + existed, err := utils.FileExist(forbiddenPagePath) + if err != nil { + logger.Error(err) + return + } + if !existed { + err = utils.EnsureWriteFile(forbiddenPagePath, []byte(constants.DefaultForbiddenPage), 0644) + if err != nil { + logger.Error(err) + } + return + } + + content, err := ioutil.ReadFile(forbiddenPagePath) + if err != nil { + logger.Error(err) + return + } + + hash := md5.New() + hash.Write([]byte(content)) + forbiddenMd5 := hex.EncodeToString(hash.Sum(nil)) + if forbiddenMd5 == constants.DefaultForbiddenPageMd5 { + return + } + + err = ioutil.WriteFile(forbiddenPagePath, []byte(constants.DefaultForbiddenPage), 0644) + if err != nil { + logger.Error(err) + return + } +} diff --git a/management/tcontrollerd/pkg/log/log.go b/management/tcontrollerd/pkg/log/log.go new file mode 100644 index 0000000..8b5639f --- /dev/null +++ b/management/tcontrollerd/pkg/log/log.go @@ -0,0 +1,117 @@ +package log + +import ( + "fmt" + "os" + "runtime" + "strings" + + "github.com/sirupsen/logrus" + + "chaitin.cn/dev/go/log" + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/config" + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/utils" +) + +func GetLogger(name string) *log.Logger { + return log.GetLogger(name) +} + +func LoadLogLevel() { + lv, _ := log.ParseLevel(config.GlobalConfig.Log.Level) + log.SetLevel(log.AllLoggers, lv) +} + +func SetLogFormatter() { + // format + formatter := new(log.TextFormatter) + formatter.FullTimestamp = true + formatter.TimestampFormat = "2006/01/02 15:04:05" + log.SetFormatter(log.AllLoggers, formatter) +} + +func InitLogger() error { + // output + switch config.GlobalConfig.Log.Output { + case "stdout": + log.SetOutput(log.AllLoggers, os.Stdout) + case "stderr": + log.SetOutput(log.AllLoggers, os.Stderr) + default: + exist, err := utils.FileExist(config.GlobalConfig.Log.Output) + if err != nil { + return err + } + + fileFlag := os.O_WRONLY | os.O_APPEND | os.O_SYNC + if !exist { + if err := utils.EnsureFileDir(config.GlobalConfig.Log.Output); err != nil { + return err + } + fileFlag = fileFlag | os.O_CREATE + } + + if fp, err := os.OpenFile(config.GlobalConfig.Log.Output, fileFlag, os.ModePerm); err != nil { + return fmt.Errorf("failed to open log file: %s", err.Error()) + } else { + log.SetOutput(log.AllLoggers, log.NewLockOutput(fp)) + } + } + + // hook + log.AddHook(log.AllLoggers, NewRuntimeHook()) + log.AddHook(log.AllLoggers, log.NewErrorStackHook(true)) + // level + LoadLogLevel() + + return nil +} + +type RuntimeHook struct{} + +func (h *RuntimeHook) Levels() []logrus.Level { + return logrus.AllLevels +} + +func (h *RuntimeHook) Fire(entry *logrus.Entry) error { + file := "???" + funcName := "???" + line := 0 + + pc := make([]uintptr, 64) + // Skip runtime.Callers, self, and another call from logrus + n := runtime.Callers(3, pc) + if n != 0 { + pc = pc[:n] // pass only valid pcs to runtime.CallersFrames + frames := runtime.CallersFrames(pc) + + // Loop to get frames. + // A fixed number of pcs can expand to an indefinite number of Frames. + for { + frame, more := frames.Next() + if !strings.Contains(frame.File, "github.com/sirupsen/logrus") && !strings.Contains(frame.Function, "chaitin.cn/dev/go") { + file = frame.File + funcName = frame.Function + line = frame.Line + break + } + if !more { + break + } + } + } + + slices := strings.Split(file, "/") + file = slices[len(slices)-1] + + funcName = strings.ReplaceAll(funcName, "chaitin.cn", "") + + entry.Data["file"] = file + entry.Data["func"] = funcName + entry.Data["line"] = line + return nil +} + +func NewRuntimeHook() *RuntimeHook { + return &RuntimeHook{} +} diff --git a/management/tcontrollerd/pkg/ngcmd/ngcmd.go b/management/tcontrollerd/pkg/ngcmd/ngcmd.go new file mode 100644 index 0000000..cbc7244 --- /dev/null +++ b/management/tcontrollerd/pkg/ngcmd/ngcmd.go @@ -0,0 +1,43 @@ +package ngcmd + +import ( + "fmt" + "os/exec" + "strings" + + "chaitin.cn/dev/go/errors" + "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/log" +) + +var logger = log.GetLogger("ngcmd") + +// NginxConfTest exec nginx -t and return stderr +func NginxConfTest() error { + out, err := exec.Command("nginx", "-t").CombinedOutput() + logger.Debugf("nginx -t output: %v", string(out)) + if err != nil { + return errors.Wrap(err, string(out)) + } + + //logger.Debugf("nginx -t output: %v", out) + if strings.Contains(string(out), "syntax is ok") && strings.Contains(string(out), "test is successful") { + return nil + } else { + return errors.New(fmt.Sprintf("nginx conf test error: %s", string(out))) + } +} + +// NginxConfReload exec nginx -t and return stderr +func NginxConfReload() error { + out, err := exec.Command("nginx", "-s", "reload").CombinedOutput() + logger.Debugf("nginx -s reload output: %v", string(out)) + if err != nil { + return errors.Wrap(err, string(out)) + } + + if len(out) == 0 { + return nil + } else { + return errors.New(fmt.Sprintf("nginx conf reload error: %s", string(out))) + } +} diff --git a/management/tcontrollerd/proto/website/website.proto b/management/tcontrollerd/proto/website/website.proto new file mode 100644 index 0000000..d272481 --- /dev/null +++ b/management/tcontrollerd/proto/website/website.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; +package website; +option go_package = "proto/website"; + +service Website { + rpc Subscribe(stream Response) returns (stream Event) {} +} + +// From client-side, may be "pong" +message Response { + string type = 1; + bytes msg = 2; + bool err = 3; +} + +// From server-side, may be "ping" +message Event { + string type = 1; // ping/website + bytes msg = 2; +} diff --git a/management/tcontrollerd/utils/file.go b/management/tcontrollerd/utils/file.go new file mode 100644 index 0000000..e4b1121 --- /dev/null +++ b/management/tcontrollerd/utils/file.go @@ -0,0 +1,126 @@ +package utils + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" +) + +func EnsureDir(dir string) error { + if _, err := os.Stat(dir); os.IsNotExist(err) { + return os.MkdirAll(dir, os.FileMode(0755)) + } + return nil +} + +func EnsureFileDir(path string) error { + return EnsureDir(filepath.Dir(path)) +} + +func FileExist(path string) (bool, error) { + stat, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } else { + return false, err + } + } else { + if stat.IsDir() { + return false, fmt.Errorf("%s is dir", path) + } else { + return true, nil + } + } +} + +func FilesExist(paths ...string) (bool, error) { + for _, path := range paths { + exist, err := FileExist(path) + if err != nil { + return false, err + } + if !exist { + return false, nil + } + } + return true, nil +} + +func RenameWriteFile(filename string, data []byte, perm os.FileMode) error { + randFileName := filename + ".tmp." + RandStr(8) + if err := ioutil.WriteFile(randFileName, data, perm); err != nil { + return err + } + return os.Rename(randFileName, filename) +} + +func EnsureRenameWriteFile(path string, data []byte, mode os.FileMode) error { + err := EnsureFileDir(path) + if err != nil { + return err + } + + return RenameWriteFile(path, data, mode) +} + +func EnsureWriteFile(path string, data []byte, mode os.FileMode) error { + err := EnsureFileDir(path) + if err != nil { + return err + } + + return ioutil.WriteFile(path, data, mode) +} + +func CopyFile(srcPath, dstPath string) error { + srcFile, err := os.Open(srcPath) + if err != nil { + return err + } + defer func(srcFile *os.File) { + err := srcFile.Close() + if err != nil { + + } + }(srcFile) + + fileInfo, err := srcFile.Stat() + if err != nil { + return err + } + + return CopyFileFromIO(srcFile, dstPath, fileInfo.Mode()) +} + +func CopyFileFromIO(src io.Reader, dstPath string, perm os.FileMode) error { + if err := EnsureFileDir(dstPath); err != nil { + return err + } + + dstFile, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, perm) + if err != nil { + return err + } + defer func(dstFile *os.File) { + err := dstFile.Close() + if err != nil { + + } + }(dstFile) + + _, err = io.Copy(dstFile, src) + return err +} + +func CopyFileIfNotExist(srcPath, dstPath string) error { + if exist, err := FileExist(dstPath); err != nil { + return err + } else if !exist { + return CopyFile(srcPath, dstPath) + } else { + return nil + } +} diff --git a/management/tcontrollerd/utils/random.go b/management/tcontrollerd/utils/random.go new file mode 100644 index 0000000..44ebd98 --- /dev/null +++ b/management/tcontrollerd/utils/random.go @@ -0,0 +1,18 @@ +package utils + +import ( + "math/rand" + "time" +) + +var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + +func RandStr(n int) string { + rand.Seed(time.Now().UnixNano()) + + b := make([]rune, n) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) +} diff --git a/management/webserver/.gitignore b/management/webserver/.gitignore new file mode 100644 index 0000000..2314a38 --- /dev/null +++ b/management/webserver/.gitignore @@ -0,0 +1,3 @@ +tmp +Taskfile.yml +.air.toml diff --git a/management/webserver/README.md b/management/webserver/README.md new file mode 100644 index 0000000..f4a7989 --- /dev/null +++ b/management/webserver/README.md @@ -0,0 +1,63 @@ +# Web Server + +web server for mgt-api + +## Requirements + +Go 1.18+ + +## Development + +### Init protobuf + +```shell +# Refer: https://grpc.io/docs/languages/go/quickstart/ +# 1. Install protoc +# https://grpc.io/docs/protoc-installation/ + +# 2. Install Go plugins +go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30.0 +go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0 + +# 3. Update your PATH so that the protoc compiler can find the plugins +# export PATH="$PATH:$(go env GOPATH)/bin" + +# 4. Generate proto go code +# cd /path/to/management +./scripts/genproto.sh +``` + +### Init fvm libs + +```shell +# Due to the fvm c header files +mkdir -p management/webserver/submodule/fvm/ +mkdir -p management/webserver/submodule/libct/ + +cd management/webserver/submodule/fvm/ +# Download https://chaitin.cn/patronus/fvm/-/tags 1.8.21 release:release, https://chaitin.cn/patronus/fvm/-/jobs/6716645 +unzip artifacts.zip +rm artifacts.zip + +cd management/webserver/submodule/libct/ +# Download https://chaitin.cn/patronus/libct/-/tags 1.1.1.0 release, https://chaitin.cn/patronus/libct/-/jobs/7229201 +# rename +rm artifacts.zip + +cd management/webserver/submodule/ +# Download https://chaitin.cn/patronus/fusion-2/-/tags 5.3.9-r1 build:release, https://chaitin.cn/patronus/fusion-2/-/jobs/7326007 +# rename +unzip artifacts.zip +mv artifacts/lib/libfusion.so libfvm.so +rm artifacts.zip +rm -r artifacts/ +``` + +### Build + +```shell +cd management/ +docker run -it --rm -w="/mnt" --mount type=bind,source="$(pwd)",target=/mnt chaitin.cn/ci/golang:1.18 bash +cp webserver/submodule/libfvm.so /usr/lib/ +make build-webserver +``` \ No newline at end of file diff --git a/management/webserver/api/auth.go b/management/webserver/api/auth.go new file mode 100644 index 0000000..96022f0 --- /dev/null +++ b/management/webserver/api/auth.go @@ -0,0 +1,186 @@ +package api + +import ( + "math" + "net/http" + "time" + + "github.com/gin-contrib/sessions" + "github.com/gin-gonic/gin" + "github.com/pquerna/otp" + "github.com/pquerna/otp/totp" + + "chaitin.cn/patronus/safeline-2/management/webserver/api/response" + "chaitin.cn/patronus/safeline-2/management/webserver/model" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/log" +) + +var logger = log.GetLogger("api") + +var OtpOpts = totp.GenerateOpts{ + Issuer: constants.ProductName, + AccountName: constants.SuperUser, + Period: 30, // seconds + Digits: otp.DigitsSix, + Algorithm: otp.AlgorithmSHA1, +} + +type PostLoginRequest struct { + Password string `json:"password"` + Passcode string `json:"passcode"` + Timestamp int64 `json:"timestamp"` +} + +func PostLogin(c *gin.Context) { + var params PostLoginRequest + if err := c.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + db := database.GetDB() + + // only SuperUser in v0.9 + var user model.User + db.Where(&model.User{Username: constants.SuperUser}).First(&user) + + // Password authentication (new) + if params.Password != "" { + if !user.CheckPassword(params.Password) { + response.Error(c, response.JSONBody{Err: response.ErrWrongPasscode, Msg: "Invalid password"}, http.StatusUnauthorized) + return + } + + // If TOTP is enabled, also require passcode (2FA) + if user.TFAEnabled && user.TFASecret != "" { + if params.Passcode == "" { + response.Error(c, response.JSONBody{Err: "totp_required", Msg: "TOTP verification required"}, http.StatusUnauthorized) + return + } + valid := totp.Validate(params.Passcode, user.TFASecret) + if !valid { + // Check clock skew + millisecondTimestamp := params.Timestamp + localTimeStamp := time.Now() + if millisecondTimestamp > 0 { + otpTime := time.Unix(millisecondTimestamp/1000, (millisecondTimestamp%1000)*int64(time.Millisecond)) + timeSub := localTimeStamp.Sub(otpTime) + seconds := math.Abs(timeSub.Seconds()) + if seconds >= 60 { + response.Error(c, response.JSONBody{Err: response.ErrWrongTimeGap, Msg: "otp timestamp gap is more than a minute"}, http.StatusUnauthorized) + return + } + } else { + response.Error(c, response.JSONBody{Err: response.ErrWrongPasscode, Msg: "Invalid TOTP passcode"}, http.StatusUnauthorized) + return + } + } + } + + // Password auth success + user.LastLoginTime = time.Now().Unix() + user.IsEnabled = true + db.Save(&user) + + session := sessions.Default(c) + session.Options(sessions.Options{ + Path: "/", + MaxAge: 3600 * 24 * 7, + }) + session.Set(constants.DefaultSessionUserKey, user.ID) + if err := session.Save(); err != nil { + logger.Error(err) + response.Error(c, response.JSONBody{Err: response.ErrInternalError, Msg: "Error occurred when creating sessions"}, http.StatusInternalServerError) + return + } + + response.Success(c, gin.H{"role": user.Role}) + return + } + + // Legacy TOTP-only authentication + if params.Passcode != "" { + valid := totp.Validate(params.Passcode, user.TFASecret) + if !valid { + millisecondTimestamp := params.Timestamp + localTimeStamp := time.Now() + if millisecondTimestamp > 0 { + otpTime := time.Unix(millisecondTimestamp/1000, (millisecondTimestamp%1000)*int64(time.Millisecond)) + timeSub := localTimeStamp.Sub(otpTime) + seconds := math.Abs(timeSub.Seconds()) + if seconds >= 60 { + response.Error(c, response.JSONBody{Err: response.ErrWrongTimeGap, Msg: "otp timestamp gap is more than a minute"}, http.StatusUnauthorized) + return + } + } + response.Error(c, response.JSONBody{Err: response.ErrWrongPasscode, Msg: "Failed to verify your passcode"}, http.StatusUnauthorized) + return + } + + user.LastLoginTime = time.Now().Unix() + user.IsEnabled = true + db.Save(&user) + + session := sessions.Default(c) + session.Options(sessions.Options{ + Path: "/", + MaxAge: 3600 * 24 * 7, + }) + session.Set(constants.DefaultSessionUserKey, user.ID) + if err := session.Save(); err != nil { + logger.Error(err) + response.Error(c, response.JSONBody{Err: response.ErrInternalError, Msg: "Error occurred when creating sessions"}, http.StatusInternalServerError) + return + } + + response.Success(c, gin.H{"role": user.Role}) + return + } + + response.Error(c, response.JSONBody{Err: response.ErrorParamNotOK, Msg: "Password or passcode required"}, http.StatusBadRequest) +} + +func PostLogout(c *gin.Context) { + session := sessions.Default(c) + session.Clear() + + if err := session.Save(); err != nil { + response.Error(c, response.JSONBody{Err: response.ErrInternalError, Msg: "Error occurred when creating sessions"}, http.StatusInternalServerError) + return + } + response.Success(c, nil) +} + +func GetOTPUrl(c *gin.Context) { + otpKey, err := totp.Generate(OtpOpts) + if err != nil { + logger.Error(err) + response.Error(c, response.JSONBody{Err: response.ErrInternalError, Msg: "Error occurred when generating otp qrcode"}, http.StatusInternalServerError) + return + } + + db := database.GetDB() + + // only SuperUser in v0.9 + user := model.User{Username: constants.SuperUser} + db.First(&user) + if user.LastLoginTime > 0 { + // already bind tfa, because tfa binding is mandatory when login. + response.Success(c, gin.H{"url": ""}) + return + } + + user.TFASecret = otpKey.Secret() + db.Save(&user) + response.Success(c, gin.H{"url": otpKey.URL()}) +} + +func GetUser(c *gin.Context) { + db := database.GetDB() + user := model.User{Username: constants.SuperUser} + db.First(&user) + response.Success(c, gin.H{"id": user.ID, "username": user.Username, "role": user.Role}) +} diff --git a/management/webserver/api/behaviour.go b/management/webserver/api/behaviour.go new file mode 100644 index 0000000..a6db3cb --- /dev/null +++ b/management/webserver/api/behaviour.go @@ -0,0 +1,25 @@ +package api + +import ( + "net/http" + + "github.com/gin-gonic/gin" + + "chaitin.cn/patronus/safeline-2/management/webserver/api/response" +) + +type PostBehaviourRequest struct { + SrcRouter string `json:"src_router"` + DstRouter string `json:"dst_router"` +} + +func PostBehaviour(c *gin.Context) { + var params PostBehaviourRequest + if err := c.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + // Behaviour tracking disabled - just return success + response.Success(c, nil) +} diff --git a/management/webserver/api/cert.go b/management/webserver/api/cert.go new file mode 100644 index 0000000..890104a --- /dev/null +++ b/management/webserver/api/cert.go @@ -0,0 +1,106 @@ +package api + +import ( + "crypto/x509/pkix" + "fmt" + "net/http" + "path/filepath" + + "github.com/gin-gonic/gin" + + "chaitin.cn/patronus/safeline-2/management/webserver/api/response" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" + "chaitin.cn/patronus/safeline-2/management/webserver/utils" +) + +type postSSLCertRequest struct { + Hostname string `json:"hostname"` +} + +// SSLCertDir is the dir of tengine conf, not mgt-api nginx certs dir defined by constants.CertsPath +const ( + CRT = ".crt" + PEM = ".pem" + KEY = ".key" + + SSLCertDir = "certs" +) + +func PostUploadSSLCert(c *gin.Context) { + file, err := c.FormFile("file") + if err != nil { + logger.Error(err) + response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + switch filepath.Ext(file.Filename) { + case CRT, PEM, KEY: + logger.Debugf("File: %v is valid", file.Filename) + default: + logger.Errorf("Filename: %s, ext: %s", file.Filename, filepath.Ext(file.Filename)) + response.Error(c, response.JSONBody{Err: response.ErrWrongFileType, Msg: "Wrong file type, please upload a file of .crt or .key"}, http.StatusUnsupportedMediaType) + return + } + + var dstPath string + filename := fmt.Sprintf("%s_%s", utils.RandStr(16), file.Filename) + if config.GlobalConfig.Server.DevMode { + dstPath = filepath.Join("./nginx", SSLCertDir, filename) + } else { + dstPath = filepath.Join(config.GlobalConfig.NgxResDir, SSLCertDir, filename) + } + if err = c.SaveUploadedFile(file, dstPath); err != nil { + logger.Error(err) + response.Error(c, response.JSONBody{Err: response.ErrInternalError, Msg: "Error occurred when saving file"}, http.StatusInternalServerError) + return + } + + response.Success(c, gin.H{"filename": filename}) +} + +func PostSSLCert(c *gin.Context) { + var params postSSLCertRequest + if err := c.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + filePrefix := utils.RandStr(16) + certFilename := fmt.Sprintf("%s_backend.crt", filePrefix) + keyFilename := fmt.Sprintf("%s_backend.key", filePrefix) + + var certPath, keyPath string + if config.GlobalConfig.Server.DevMode { + certPath = filepath.Join("./management", SSLCertDir, certFilename) + keyPath = filepath.Join("./management", SSLCertDir, keyFilename) + } else { + certPath = filepath.Join(config.GlobalConfig.NgxResDir, SSLCertDir, certFilename) + keyPath = filepath.Join(config.GlobalConfig.NgxResDir, SSLCertDir, keyFilename) + } + if err := utils.WriteCertIfNotExist( + certPath, + keyPath, + func() ([]byte, []byte, error) { + return utils.GenerateCert( + []string{params.Hostname}, + 3650, + 4096, + &pkix.Name{ + Country: []string{}, + Province: []string{}, + Locality: []string{}, + Organization: []string{}, + OrganizationalUnit: []string{}, + CommonName: "", + }, + false, + ) + }); err != nil { + logger.Error(err) + response.Error(c, response.JSONBody{Err: response.ErrInternalError, Msg: "Error occurred when generating certs"}, http.StatusInternalServerError) + return + } + + response.Success(c, gin.H{"crt": certFilename, "key": keyFilename}) +} diff --git a/management/webserver/api/common.go b/management/webserver/api/common.go new file mode 100644 index 0000000..1197e45 --- /dev/null +++ b/management/webserver/api/common.go @@ -0,0 +1,26 @@ +package api + +import ( + "github.com/gin-gonic/gin" + + "chaitin.cn/patronus/safeline-2/management/webserver/api/response" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" +) + +type idsRequest struct { + IDs []uint `json:"ids" form:"ids"` +} + +type pageRequest struct { + Page int `json:"page" form:"page,default=1" binding:"min=1"` + PageSize int `json:"page_size" form:"page_size,default=10" binding:"min=1"` +} + +func GetVersion(c *gin.Context) { + response.Success(c, gin.H{"version": constants.Version}) +} + +func GetUpgradeTips(ctx *gin.Context) { + // External version check disabled - always return no upgrade needed + response.Success(ctx, gin.H{"upgrade_tips": constants.NotUpgrade}) +} diff --git a/management/webserver/api/dashboard.go b/management/webserver/api/dashboard.go new file mode 100644 index 0000000..3ecda2f --- /dev/null +++ b/management/webserver/api/dashboard.go @@ -0,0 +1,117 @@ +package api + +import ( + "math" + "time" + + "github.com/gin-gonic/gin" + + "chaitin.cn/patronus/safeline-2/management/webserver/api/response" + "chaitin.cn/patronus/safeline-2/management/webserver/model" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" +) + +func GetDashboardCounts(c *gin.Context) { + var requested int64 = 0 + var intercepted int64 = 0 + db := database.GetDB() + + var statisticTotalReq model.SystemStatistics + r := db.Where("type = 'total-req'").Where("created_at >= date_trunc('day',now())").First(&statisticTotalReq) + if r.RowsAffected > 0 { + requested = statisticTotalReq.Value + } + + var statisticTotalDenied model.SystemStatistics + r = db.Where("type = 'total-denied'").Where("created_at >= date_trunc('day',now())").First(&statisticTotalDenied) + if r.RowsAffected > 0 { + intercepted = statisticTotalDenied.Value + } + response.Success(c, gin.H{"requested": requested, "intercepted": intercepted}) +} + +func GetDashboardSites(c *gin.Context) { + response.Success(c, gin.H{"normal": 0, "abnormal": 0}) +} + +func GetDashboardQps(c *gin.Context) { + + var statistics []model.SystemStatistics + + db := database.GetDB() + db.Where("type = 'req'").Order("created_at desc").Limit(75).Find(&statistics) + + type Node struct { + Label string `json:"label"` + Value int64 `json:"value"` + } + var nodes = make([]Node, 0) + + for i := len(statistics) - 1; i >= 0; i-- { + nodes = append(nodes, Node{ + Label: statistics[i].CreatedAt.Format("2006-01-02 15:04:05"), + Value: int64(math.Ceil(float64(statistics[i].Value) / 5)), + }) + } + + response.Success(c, gin.H{"nodes": nodes, "total": len(nodes)}) +} + +func GetDashboardRequests(c *gin.Context) { + + var statistics []model.SystemStatistics + + db := database.GetDB() + db.Where("type = 'total-req'").Order("created_at desc").Limit(30).Find(&statistics) + + type Node struct { + Label string `json:"label"` + Value int64 `json:"value"` + } + var nodes = make([]Node, 0) + + for i := 30; i > len(statistics); i-- { + nodes = append(nodes, Node{ + Label: time.Now().Add(-time.Duration(i-1) * time.Hour * 24).Format("2006-01-02"), + Value: 0, + }) + } + + for i := len(statistics) - 1; i >= 0; i-- { + nodes = append(nodes, Node{ + Label: statistics[i].CreatedAt.Format("2006-01-02"), + Value: statistics[i].Value, + }) + } + + response.Success(c, gin.H{"nodes": nodes, "total": len(nodes)}) +} + +func GetDashboardIntercepts(c *gin.Context) { + var statistics []model.SystemStatistics + + db := database.GetDB() + db.Where("type = 'total-denied'").Order("created_at desc").Limit(30).Find(&statistics) + + type Node struct { + Label string `json:"label"` + Value int64 `json:"value"` + } + var nodes = make([]Node, 0) + + for i := 30; i > len(statistics); i-- { + nodes = append(nodes, Node{ + Label: time.Now().Add(-time.Duration(i-1) * time.Hour * 24).Format("2006-01-02"), + Value: 0, + }) + } + + for i := len(statistics) - 1; i >= 0; i-- { + nodes = append(nodes, Node{ + Label: statistics[i].CreatedAt.Format("2006-01-02"), + Value: statistics[i].Value, + }) + } + + response.Success(c, gin.H{"nodes": nodes, "total": len(nodes)}) +} diff --git a/management/webserver/api/detectlog.go b/management/webserver/api/detectlog.go new file mode 100644 index 0000000..4e3993b --- /dev/null +++ b/management/webserver/api/detectlog.go @@ -0,0 +1,121 @@ +package api + +import ( + "net/http" + "strconv" + "strings" + + "chaitin.cn/dev/go/errors" + + "github.com/gin-gonic/gin" + + "chaitin.cn/patronus/safeline-2/management/webserver/api/response" + "chaitin.cn/patronus/safeline-2/management/webserver/model" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" +) + +type ( + GetDetectLogDetailRequest struct { + EventId string `json:"event_id" form:"event_id"` + } + + PostFalsePositivesRequest struct { + EventId string `json:"event_id"` + } +) + +func getDetectLog(eventId string) (*model.DetectLog, error) { + db := database.GetDB() + var detectLogBasic model.DetectLogBasic + res := db.Where(&model.DetectLogBasic{EventId: eventId}).First(&detectLogBasic) + if res.RowsAffected == 0 { + return nil, errors.New("Data queried does not exist") + } + + var detectLogDetail model.DetectLogDetail + db.Where(&model.DetectLogDetail{EventId: eventId}).First(&detectLogDetail) + + detectLog, err := model.TransformDetectLog(&detectLogBasic, &detectLogDetail) + if err != nil { + return nil, err + } + + return detectLog, nil +} + +func GetDetectLogList(c *gin.Context) { + var params pageRequest + if err := c.BindQuery(¶ms); err != nil { + logger.Error(err) + response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + db := database.GetDB() + + tx := db.Where("") + // 按 ip 搜索条件 + if ip := c.Query("ip"); ip != "" { + tx = tx.Where("src_ip = ?", ip) + } + // 按 url 搜索条件 + if url := c.Query("url"); url != "" { + tx = tx.Where("url_path like ?", "%"+url+"%") + } + // 按 type 搜索条件 + if at := c.Query("attack_type"); at != "" { + ns := make([]int, 0) + for _, s := range strings.Split(at, ",") { + n, err := strconv.Atoi(s) + if err == nil { + ns = append(ns, n) + } + } + tx = tx.Where("attack_type in (?)", ns) + } + + var total int64 + tx.Model(&model.DetectLogBasic{}).Count(&total) + + var basicList []model.DetectLogBasic + tx.Limit(params.PageSize).Offset(params.PageSize * (params.Page - 1)).Order("id desc").Find(&basicList) + var dLogList []*model.DetectLog + for _, basic := range basicList { + dLog, err := model.TransformDetectLog(&basic, nil) + if err != nil { + logger.Warn(err) + continue + } + dLogList = append(dLogList, dLog) + } + response.Success(c, gin.H{"data": dLogList, "total": total}) +} + +func GetDetectLogDetail(c *gin.Context) { + var params GetDetectLogDetailRequest + if err := c.BindQuery(¶ms); err != nil { + logger.Error(err) + response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + detectLog, err := getDetectLog(params.EventId) + if err != nil { + logger.Error(err) + response.Error(c, response.ErrorDataNotExist, http.StatusNotFound) + return + } + + response.Success(c, detectLog) +} + +func PostFalsePositives(c *gin.Context) { + var params PostFalsePositivesRequest + if err := c.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + // Telemetry reporting removed - just return success + response.Success(c, nil) +} diff --git a/management/webserver/api/endpoints.go b/management/webserver/api/endpoints.go new file mode 100644 index 0000000..993ae25 --- /dev/null +++ b/management/webserver/api/endpoints.go @@ -0,0 +1,29 @@ +package api + +const ( + Version = "/Version" + UpgradeTips = "/UpgradeTips" + Login = "/Login" + Logout = "/Logout" + OTPUrl = "/OTPUrl" + User = "/User" + CurrentUser = "/CurrentUser" + Users = "/Users" + DetectLogList = "/DetectLogList" + DetectLogDetail = "/DetectLogDetail" + Behaviour = "/Behaviour" + FalsePositives = "/FalsePositives" + Website = "/Website" + UploadSSLCert = "/UploadSSLCert" + SSLCert = "/SSLCert" + PolicyRule = "/PolicyRule" + SwitchPolicyRule = "/SwitchPolicyRule" + DashboardCounts = "/dashboard/counts" + DashboardSites = "/dashboard/sites" + DashboardQps = "/dashboard/qps" + DashboardRequests = "/dashboard/requests" + DashboardIntercepts = "/dashboard/intercepts" + PolicyGroupGlobal = "/PolicyGroupGlobal" + PolicyGroupPreset = "/PolicyGroupPreset" + SrcIPConfig = "/SrcIPConfig" +) diff --git a/management/webserver/api/policygroup.go b/management/webserver/api/policygroup.go new file mode 100644 index 0000000..4086afa --- /dev/null +++ b/management/webserver/api/policygroup.go @@ -0,0 +1,185 @@ +package api + +import ( + "encoding/json" + "net/http" + + "github.com/gin-gonic/gin" + "gorm.io/gorm" + + "chaitin.cn/dev/go/errors" + + "chaitin.cn/patronus/safeline-2/management/webserver/api/response" + "chaitin.cn/patronus/safeline-2/management/webserver/model" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/fvm" +) + +func PutPolicyGroupGlobal(ctx *gin.Context) { + var params model.PolicyGroup + if err := ctx.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + db := database.GetDB() + err := db.Transaction(func(tx *gorm.DB) error { + var pggOption model.Options + res := tx.Where(&model.Options{Key: constants.PolicyGroupGlobal}).First(&pggOption) + if res.Error != nil { + return res.Error + } + if res.RowsAffected == 0 { + return errors.New("Data queried does not exist") + } + + pggStr, err := json.Marshal(params) + if err != nil { + logger.Error(err) + response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) + return err + } + + pggOption.Value = string(pggStr) + tx.Save(&pggOption) + + if err := fvm.PushFSL(tx); err != nil { + return errors.New("Rules compile error, please check your params.") + } + + return nil + }) + if err != nil { + logger.Error(err) + response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) + return + } + + response.Success(ctx, nil) +} + +func GetPolicyGroupGlobal(ctx *gin.Context) { + var pggOption model.Options + database.GetDB().Where(&model.Options{Key: constants.PolicyGroupGlobal}).First(&pggOption) + + var pgg model.PolicyGroup + err := json.Unmarshal([]byte(pggOption.Value), &pgg) + if err != nil { + logger.Error(err) + response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) + return + } + + response.Success(ctx, gin.H{"data": pgg}) +} + +func PutSrcIPConfig(ctx *gin.Context) { + var params model.SrcIPConfig + if err := ctx.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + db := database.GetDB() + err := db.Transaction(func(tx *gorm.DB) error { + var scOption model.Options + res := tx.Where(&model.Options{Key: constants.SrcIPConfig}).First(&scOption) + if res.Error != nil { + return res.Error + } + if res.RowsAffected == 0 { + return errors.New("Data queried does not exist") + } + + scStr, err := json.Marshal(params) + if err != nil { + logger.Error(err) + response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) + return err + } + + scOption.Value = string(scStr) + tx.Save(&scOption) + + if err := fvm.PushFSL(tx); err != nil { + return errors.New("Rules compile error, please check your params.") + } + + return nil + }) + if err != nil { + logger.Error(err) + response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) + return + } + + response.Success(ctx, nil) +} + +func GetSrcIPConfig(ctx *gin.Context) { + srcIPConfig, err := model.GetSrcIPConfig(database.GetDB().DB) + if err != nil { + return + } + response.Success(ctx, gin.H{"data": srcIPConfig}) +} + +type PolicyGroupPresetRequest struct { + Preset string `json:"preset" binding:"required,oneof=strict default disable"` +} + +func PostPolicyGroupPreset(ctx *gin.Context) { + var params PolicyGroupPresetRequest + if err := ctx.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + // All 17 detection modules + modules := []string{ + "m_asp_code_injection", "m_cmd_injection", "m_csrf", + "m_file_include", "m_file_upload", "m_http", + "m_java", "m_java_unserialize", "m_php_code_injection", + "m_php_unserialize", "m_response", "m_rule", + "m_scanner", "m_sqli", "m_ssrf", "m_ssti", "m_xss", + } + + pgg := make(model.PolicyGroup) + for _, m := range modules { + pgg[m] = params.Preset + } + + db := database.GetDB() + err := db.Transaction(func(tx *gorm.DB) error { + var pggOption model.Options + res := tx.Where(&model.Options{Key: constants.PolicyGroupGlobal}).First(&pggOption) + if res.Error != nil { + return res.Error + } + + pggStr, err := json.Marshal(pgg) + if err != nil { + return err + } + + pggOption.Value = string(pggStr) + tx.Save(&pggOption) + + if err := fvm.PushFSL(tx); err != nil { + return errors.New("Rules compile error") + } + + return nil + }) + if err != nil { + logger.Error(err) + response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) + return + } + + response.Success(ctx, gin.H{"preset": params.Preset, "data": pgg}) +} diff --git a/management/webserver/api/policyrule.go b/management/webserver/api/policyrule.go new file mode 100644 index 0000000..784b427 --- /dev/null +++ b/management/webserver/api/policyrule.go @@ -0,0 +1,174 @@ +package api + +import ( + "net/http" + + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/fvm" + + "github.com/gin-gonic/gin" + "gorm.io/gorm" + + "chaitin.cn/dev/go/errors" + + "chaitin.cn/patronus/safeline-2/management/webserver/api/response" + "chaitin.cn/patronus/safeline-2/management/webserver/model" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" +) + +type putSwitchRequest struct { + IDs []uint `json:"ids"` + IsEnabled bool `json:"is_enabled"` +} + +func PostPolicyRule(ctx *gin.Context) { + var params model.PolicyRule + if err := ctx.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + db := database.GetDB() + err := db.Transaction(func(tx *gorm.DB) error { + policyRule := &model.PolicyRule{Action: params.Action, Comment: params.Comment, IsEnabled: params.IsEnabled, Pattern: params.Pattern} + res := tx.Create(policyRule) + if res.Error != nil { + return res.Error + } + + if err := fvm.PushFSL(tx); err != nil { + return errors.New("Rules compile error, please check your params.") + } + + return nil + }) + if err != nil { + logger.Error(err) + response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) + return + } + + response.Success(ctx, nil) +} + +func PutSwitchPolicyRule(ctx *gin.Context) { + var params putSwitchRequest + if err := ctx.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + db := database.GetDB() + err := db.Transaction(func(tx *gorm.DB) error { + res := tx.Model(&model.PolicyRule{}).Where(params.IDs).Updates(model.PolicyRule{IsEnabled: params.IsEnabled}) + if res.Error != nil { + return res.Error + } + if res.RowsAffected == 0 { + return errors.New("Data queried does not exist") + } + + if err := fvm.PushFSL(tx); err != nil { + return errors.New("Rules compile error, please check your params.") + } + + return nil + }) + if err != nil { + logger.Error(err) + response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) + return + } +} + +func PutPolicyRule(ctx *gin.Context) { + var params model.PolicyRule + if err := ctx.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + db := database.GetDB() + err := db.Transaction(func(tx *gorm.DB) error { + var policyRule model.PolicyRule + res := tx.Where(params.ID).First(&policyRule) + if res.Error != nil { + return res.Error + } + if res.RowsAffected == 0 { + return errors.New("Data queried does not exist") + } + + policyRule.Action = params.Action + policyRule.Comment = params.Comment + policyRule.IsEnabled = params.IsEnabled + policyRule.Pattern = params.Pattern + tx.Save(&policyRule) + + if err := fvm.PushFSL(tx); err != nil { + return errors.New("Rules compile error, please check your params.") + } + + return nil + }) + if err != nil { + logger.Error(err) + response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) + return + } + + response.Success(ctx, nil) +} + +func DeletePolicyRule(ctx *gin.Context) { + var params idsRequest + if err := ctx.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + db := database.GetDB() + err := db.Transaction(func(tx *gorm.DB) error { + res := tx.Where(params.IDs).Delete(&model.PolicyRule{}) + if res.Error != nil { + return res.Error + } + if res.RowsAffected == 0 { + return errors.New("Data queried does not exist") + } + + if err := fvm.PushFSL(tx); err != nil { + return errors.New("Rules compile error, please check your params.") + } + + return nil + }) + if err != nil { + logger.Error(err) + response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) + return + } + + response.Success(ctx, nil) +} + +func GetPolicyRule(ctx *gin.Context) { + var params pageRequest + if err := ctx.BindQuery(¶ms); err != nil { + logger.Error(err) + response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + db := database.GetDB() + var policyRuleList []model.PolicyRule + db.Limit(params.PageSize).Offset(params.PageSize * (params.Page - 1)).Order("id desc").Find(&policyRuleList) + + var total int64 + db.Model(&model.PolicyRule{}).Count(&total) + + response.Success(ctx, gin.H{"data": policyRuleList, "total": total}) +} diff --git a/management/webserver/api/response/error.go b/management/webserver/api/response/error.go new file mode 100644 index 0000000..d62b815 --- /dev/null +++ b/management/webserver/api/response/error.go @@ -0,0 +1,11 @@ +package response + +const ( + ErrLoginRequired = "login-required" + ErrWrongPasscode = "wrong-passcode" + ErrWrongTimeGap = "wrong-time-gap" + ErrInternalError = "internal-error" + ErrDataNotExist = "data-not-exist" + ErrWrongFileType = "wrong-filetype" + ErrReadOnly = "read-only" +) diff --git a/management/webserver/api/response/jsonbody.go b/management/webserver/api/response/jsonbody.go new file mode 100644 index 0000000..4af6e61 --- /dev/null +++ b/management/webserver/api/response/jsonbody.go @@ -0,0 +1,56 @@ +package response + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +type JSONBody struct { + Err string + Msg string + Data interface{} +} + +var ( + ErrorLoginRequired = JSONBody{ErrLoginRequired, "Login required", nil} + ErrorParamNotOK = JSONBody{ErrInternalError, "Error occurred when extracting params", nil} + ErrorDataNotExist = JSONBody{ErrDataNotExist, "Data queried does not exist", nil} + ErrorReadOnly = JSONBody{ErrReadOnly, "This environment is read only", nil} +) + +func Success(c *gin.Context, data interface{}) { + c.JSON(http.StatusOK, gin.H{ + "data": data, + "msg": "", + "err": nil, + }) + //c.Abort() +} + +func SuccessWithList(c *gin.Context, data interface{}) { + c.JSON(http.StatusOK, gin.H{ + "data": data, + "msg": "", + "err": nil, + }) + //c.Abort() +} + +func SuccessWithMsg(c *gin.Context, data interface{}, msg string) { + c.JSON(http.StatusOK, gin.H{ + "data": data, + "msg": msg, + "err": nil, + }) + //c.Abort() +} + +func Error(c *gin.Context, rsp JSONBody, status int) { + c.JSON(status, gin.H{ + "data": rsp.Data, + "msg": rsp.Msg, + "err": rsp.Err, + }) + //c.Abort() +} diff --git a/management/webserver/api/response/png.go b/management/webserver/api/response/png.go new file mode 100644 index 0000000..57327df --- /dev/null +++ b/management/webserver/api/response/png.go @@ -0,0 +1,13 @@ +package response + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +const StreamContentType = "application/octet-stream" + +func PNG(c *gin.Context, bytes []byte) { + c.Data(http.StatusOK, StreamContentType, bytes) +} diff --git a/management/webserver/api/user.go b/management/webserver/api/user.go new file mode 100644 index 0000000..fe56faa --- /dev/null +++ b/management/webserver/api/user.go @@ -0,0 +1,231 @@ +package api + +import ( + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + + "chaitin.cn/patronus/safeline-2/management/webserver/api/response" + "chaitin.cn/patronus/safeline-2/management/webserver/model" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" +) + +type CreateUserRequest struct { + Username string `json:"username" binding:"required,min=3,max=32"` + Password string `json:"password" binding:"required,min=6,max=64"` + Role string `json:"role" binding:"required,oneof=admin operator viewer"` + Comment string `json:"comment"` +} + +type UpdateUserRequest struct { + Role string `json:"role" binding:"omitempty,oneof=admin operator viewer"` + Comment string `json:"comment"` + Enabled *bool `json:"enabled"` +} + +type ResetPasswordRequest struct { + Password string `json:"password" binding:"required,min=6,max=64"` +} + +type UserInfo struct { + ID uint `json:"id"` + Username string `json:"username"` + Role string `json:"role"` + Comment string `json:"comment"` + Enabled bool `json:"enabled"` +} + +func GetUsers(c *gin.Context) { + db := database.GetDB() + var users []model.User + db.Find(&users) + + var result []UserInfo + for _, u := range users { + result = append(result, UserInfo{ + ID: u.ID, + Username: u.Username, + Role: u.Role, + Comment: u.Comment, + Enabled: u.IsEnabled, + }) + } + + response.Success(c, gin.H{"data": result}) +} + +func PostUser(c *gin.Context) { + var params CreateUserRequest + if err := c.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + db := database.GetDB() + + // Check if username already exists + var existing model.User + res := db.Where(&model.User{Username: params.Username}).First(&existing) + if res.RowsAffected > 0 { + response.Error(c, response.JSONBody{Err: "username_exists", Msg: "Username already exists"}, http.StatusConflict) + return + } + + user := model.User{ + Username: params.Username, + Role: params.Role, + Comment: params.Comment, + } + if err := user.SetPassword(params.Password); err != nil { + logger.Error(err) + response.Error(c, response.JSONBody{Err: response.ErrInternalError, Msg: "Failed to hash password"}, http.StatusInternalServerError) + return + } + + if err := db.Create(&user).Error; err != nil { + logger.Error(err) + response.Error(c, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) + return + } + + response.Success(c, gin.H{ + "id": user.ID, + "username": user.Username, + "role": user.Role, + }) +} + +func PutUser(c *gin.Context) { + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + response.Error(c, response.ErrorParamNotOK, http.StatusBadRequest) + return + } + + var params UpdateUserRequest + if err := c.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + db := database.GetDB() + var user model.User + res := db.First(&user, id) + if res.RowsAffected == 0 { + response.Error(c, response.ErrorDataNotExist, http.StatusNotFound) + return + } + + // Prevent disabling the last admin + if params.Enabled != nil && !*params.Enabled && user.Role == "admin" { + var adminCount int64 + db.Model(&model.User{}).Where("role = ? AND is_enabled = true", "admin").Count(&adminCount) + if adminCount <= 1 { + response.Error(c, response.JSONBody{Err: "last_admin", Msg: "Cannot disable the last admin user"}, http.StatusConflict) + return + } + } + + updates := map[string]interface{}{} + if params.Role != "" { + updates["role"] = params.Role + } + if params.Comment != "" { + updates["comment"] = params.Comment + } + if params.Enabled != nil { + updates["is_enabled"] = *params.Enabled + } + + db.Model(&user).Updates(updates) + response.Success(c, nil) +} + +func DeleteUser(c *gin.Context) { + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + response.Error(c, response.ErrorParamNotOK, http.StatusBadRequest) + return + } + + db := database.GetDB() + var user model.User + res := db.First(&user, id) + if res.RowsAffected == 0 { + response.Error(c, response.ErrorDataNotExist, http.StatusNotFound) + return + } + + // Prevent deleting the last admin + if user.Role == "admin" { + var adminCount int64 + db.Model(&model.User{}).Where("role = ? AND is_enabled = true", "admin").Count(&adminCount) + if adminCount <= 1 { + response.Error(c, response.JSONBody{Err: "last_admin", Msg: "Cannot delete the last admin user"}, http.StatusConflict) + return + } + } + + // Prevent self-deletion + sessionUser := c.GetInt("user_id") + if uint(sessionUser) == user.ID { + response.Error(c, response.JSONBody{Err: "self_delete", Msg: "Cannot delete your own account"}, http.StatusConflict) + return + } + + db.Delete(&user) + response.Success(c, nil) +} + +func PostResetPassword(c *gin.Context) { + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + response.Error(c, response.ErrorParamNotOK, http.StatusBadRequest) + return + } + + var params ResetPasswordRequest + if err := c.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + db := database.GetDB() + var user model.User + res := db.First(&user, id) + if res.RowsAffected == 0 { + response.Error(c, response.ErrorDataNotExist, http.StatusNotFound) + return + } + + if err := user.SetPassword(params.Password); err != nil { + logger.Error(err) + response.Error(c, response.JSONBody{Err: response.ErrInternalError, Msg: "Failed to hash password"}, http.StatusInternalServerError) + return + } + + db.Model(&user).Update("password_hash", user.PasswordHash) + response.Success(c, nil) +} + +// GetUser returns the current authenticated user info +func GetCurrentUser(c *gin.Context) { + userId, _ := c.Get("user_id") + db := database.GetDB() + var user model.User + res := db.First(&user, userId) + if res.RowsAffected == 0 { + response.Error(c, response.ErrorDataNotExist, http.StatusNotFound) + return + } + + response.Success(c, gin.H{ + "id": user.ID, + "username": user.Username, + "role": user.Role, + }) +} diff --git a/management/webserver/api/website.go b/management/webserver/api/website.go new file mode 100644 index 0000000..c21b3d4 --- /dev/null +++ b/management/webserver/api/website.go @@ -0,0 +1,211 @@ +package api + +import ( + "encoding/json" + "net/http" + "strconv" + + "chaitin.cn/dev/go/errors" + + "github.com/gin-gonic/gin" + "gorm.io/gorm" + + "chaitin.cn/patronus/safeline-2/management/webserver/api/response" + "chaitin.cn/patronus/safeline-2/management/webserver/model" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" + "chaitin.cn/patronus/safeline-2/management/webserver/rpc" +) + +func publishWebsiteConfig(website *model.Website) error { + byteWebsite, err := json.Marshal(website) + if err != nil { + return err + } + + err = rpc.Publish(byteWebsite, rpc.EventTypeWebsite) + if err != nil { + return err + } + + return nil +} + +func publishDeleteWebsiteConfig(id []uint) error { + byteId, err := json.Marshal(id) + if err != nil { + return err + } + + err = rpc.Publish(byteId, rpc.EventTypeDeleteWebsite) + if err != nil { + return err + } + + return nil +} + +func PostWebsite(ctx *gin.Context) { + var params model.Website + if err := ctx.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + db := database.GetDB() + err := db.Transaction(func(tx *gorm.DB) error { + website := &model.Website{Comment: params.Comment, ServerNames: params.ServerNames, Upstreams: params.Upstreams, Ports: params.Ports, + CertFilename: params.CertFilename, KeyFilename: params.KeyFilename, IsEnabled: true} + res := tx.Create(website) + if res.Error != nil { + return res.Error + } + + if config.GlobalConfig.Server.DevMode { + return nil + } + + err := publishWebsiteConfig(website) + if err != nil { + return err + } + + return nil + }) + if err != nil { + logger.Error(err) + response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) + return + } + + response.Success(ctx, nil) +} + +func PutWebsite(ctx *gin.Context) { + var params model.Website + if err := ctx.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + db := database.GetDB() + err := db.Transaction(func(tx *gorm.DB) error { + var website model.Website + res := tx.Where(params.ID).First(&website) + if res.Error != nil { + return res.Error + } + if res.RowsAffected == 0 { + return errors.New("Data queried does not exist") + } + + website.Comment = params.Comment + website.ServerNames = params.ServerNames + website.Upstreams = params.Upstreams + website.Ports = params.Ports + website.CertFilename = params.CertFilename + website.KeyFilename = params.KeyFilename + website.IsEnabled = true + tx.Save(&website) + + if config.GlobalConfig.Server.DevMode { + return nil + } + + err := publishWebsiteConfig(¶ms) + if err != nil { + return err + } + + return nil + }) + if err != nil { + logger.Error(err) + response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) + return + } + + response.Success(ctx, nil) +} + +func DeleteWebsite(ctx *gin.Context) { + var params idsRequest + if err := ctx.BindJSON(¶ms); err != nil { + logger.Error(err) + response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + db := database.GetDB() + err := db.Transaction(func(tx *gorm.DB) error { + res := tx.Where(params.IDs).Delete(&model.Website{}) + if res.Error != nil { + return res.Error + } + if res.RowsAffected == 0 { + return errors.New("Data queried does not exist") + } + + if config.GlobalConfig.Server.DevMode { + return nil + } + + err := publishDeleteWebsiteConfig(params.IDs) + if err != nil { + return err + } + + return nil + }) + if err != nil { + logger.Error(err) + response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) + return + } + + response.Success(ctx, nil) +} + +func GetWebsite(ctx *gin.Context) { + var params pageRequest + if err := ctx.BindQuery(¶ms); err != nil { + logger.Error(err) + response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) + return + } + + db := database.GetDB() + type Website struct { + model.Website + ReqValue int64 `json:"req_value"` + DeniedValue int64 `json:"denied_value"` + } + var websiteList []Website + db.Limit(params.PageSize).Offset(params.PageSize * (params.Page - 1)).Order("id desc").Find(&websiteList) + + var statistics []model.SystemStatistics + var websiteIds []string + for _, i := range websiteList { + websiteIds = append(websiteIds, strconv.Itoa(int(i.ID))) + } + db.Where("created_at >= date_trunc('day',now())").Where("website in (?)", websiteIds).Find(&statistics) + + for i, website := range websiteList { + for _, j := range statistics { + if strconv.Itoa(int(website.ID)) == j.Website { + if j.Type == "website-req" { + websiteList[i].ReqValue = j.Value + } else if j.Type == "website-denied" { + websiteList[i].DeniedValue = j.Value + } + } + } + } + + var total int64 + db.Model(&model.Website{}).Count(&total) + + response.Success(ctx, gin.H{"data": websiteList, "total": total}) +} diff --git a/management/webserver/cmd/fake_logs.go b/management/webserver/cmd/fake_logs.go new file mode 100644 index 0000000..dc79510 --- /dev/null +++ b/management/webserver/cmd/fake_logs.go @@ -0,0 +1,7 @@ +package cmd + +import "chaitin.cn/patronus/safeline-2/management/webserver/model" + +func FakeLogs() { + model.InitDetectLogSamples() +} diff --git a/management/webserver/cmd/gen_certs.go b/management/webserver/cmd/gen_certs.go new file mode 100644 index 0000000..509e6e4 --- /dev/null +++ b/management/webserver/cmd/gen_certs.go @@ -0,0 +1,74 @@ +package cmd + +import ( + "crypto/x509/pkix" + "path/filepath" + + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" + "chaitin.cn/patronus/safeline-2/management/webserver/utils" +) + +func GenCerts() error { + if err := genServerCert(); err != nil { + return err + } + if err := genClientCACert(); err != nil { + return err + } + return nil +} + +func genServerCert() error { + certPath := filepath.Join(config.GlobalConfig.MgtResDir, constants.CertsPath, "server.crt") + keyPath := filepath.Join(config.GlobalConfig.MgtResDir, constants.CertsPath, "server.key") + if err := utils.WriteCertIfNotExist( + certPath, + keyPath, + func() ([]byte, []byte, error) { + return utils.GenerateCert( + []string{}, + 3650, + 4096, + &pkix.Name{ + Country: []string{"CN"}, + Province: []string{"Beijing"}, + Locality: []string{"Beijing"}, + Organization: []string{"Beijing WAF Technology Co., Ltd."}, + OrganizationalUnit: []string{"Service Infrastructure Department"}, + CommonName: "WAF Management Server", + }, + false, + ) + }); err != nil { + return err + } + return nil +} + +func genClientCACert() error { + certPath := filepath.Join(config.GlobalConfig.MgtResDir, constants.CertsPath, "client_ca.crt") + keyPath := filepath.Join(config.GlobalConfig.MgtResDir, constants.CertsPath, "client_ca.key") + if err := utils.WriteCertIfNotExist( + certPath, + keyPath, + func() ([]byte, []byte, error) { + return utils.GenerateCert( + []string{}, + 3650, + 4096, + &pkix.Name{ + Country: []string{"CN"}, + Province: []string{"Beijing"}, + Locality: []string{"Beijing"}, + Organization: []string{"Beijing WAF Technology Co., Ltd."}, + OrganizationalUnit: []string{"Service Infrastructure Department"}, + CommonName: "WAF Client Certificate Authority", + }, + true, + ) + }); err != nil { + return err + } + return nil +} diff --git a/management/webserver/cmd/push_fsl.go b/management/webserver/cmd/push_fsl.go new file mode 100644 index 0000000..01c4ac0 --- /dev/null +++ b/management/webserver/cmd/push_fsl.go @@ -0,0 +1,10 @@ +package cmd + +import ( + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/fvm" +) + +func PushFSL() error { + return fvm.PushFSL(database.GetDB().DB) +} diff --git a/management/webserver/cmd/reset_user.go b/management/webserver/cmd/reset_user.go new file mode 100644 index 0000000..c879f0d --- /dev/null +++ b/management/webserver/cmd/reset_user.go @@ -0,0 +1,14 @@ +package cmd + +import ( + "chaitin.cn/patronus/safeline-2/management/webserver/model" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" +) + +func ResetUser(username string) { + db := database.GetDB() + var user model.User + db.Where(&model.User{Username: username}).First(&user) + user.LastLoginTime = 0 + db.Save(&user) +} diff --git a/management/webserver/cmd/show_fsl.go b/management/webserver/cmd/show_fsl.go new file mode 100644 index 0000000..3dc98f2 --- /dev/null +++ b/management/webserver/cmd/show_fsl.go @@ -0,0 +1,10 @@ +package cmd + +import ( + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/fvm" +) + +func ShowFSL() (string, error) { + return fvm.GenerateFullFSL(database.GetDB().DB) +} diff --git a/management/webserver/config.yml b/management/webserver/config.yml new file mode 100644 index 0000000..e560762 --- /dev/null +++ b/management/webserver/config.yml @@ -0,0 +1,16 @@ +# develop use only. For production, refer to `package/build/mgt-api/webserver/config.yml` +log: + output: stdout # "stdout", "stderr" or file path + level: debug # "debug", "info", "warn" or "error" +server: + listen_addr: :9001 + dev_mode: true +db: + url: postgres://safeline-ce:safeline-ce@127.0.0.1/safeline-ce + log_sql: false +detector: + addr: "" + fsl_bytecode: fvm/bytecode +grpc_server: + listen_addr: :9002 + diff --git a/management/webserver/go.mod b/management/webserver/go.mod new file mode 100644 index 0000000..c13ca16 --- /dev/null +++ b/management/webserver/go.mod @@ -0,0 +1,60 @@ +module chaitin.cn/patronus/safeline-2/management/webserver + +go 1.21 + +toolchain go1.21.3 + +require ( + chaitin.cn/dev/go/errors v0.0.0-20210324055134-dc5247602af6 + chaitin.cn/dev/go/log v0.0.0-20221220104336-05125760b10c + chaitin.cn/dev/go/settings v0.0.0-20221220104336-05125760b10c + github.com/gin-contrib/sessions v0.0.5 + github.com/gin-gonic/gin v1.10.0 + github.com/pquerna/otp v1.4.0 + github.com/robfig/cron/v3 v3.0.1 + github.com/rogpeppe/go-internal v1.10.0 + github.com/sirupsen/logrus v1.4.2 + golang.org/x/crypto v0.28.0 + google.golang.org/grpc v1.65.0 + gorm.io/datatypes v1.1.1 + gorm.io/driver/postgres v1.5.9 + gorm.io/gorm v1.25.10 +) + +require ( + github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect + github.com/go-sql-driver/mysql v1.7.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/gorilla/context v1.1.1 // indirect + github.com/gorilla/securecookie v1.1.1 // indirect + github.com/gorilla/sessions v1.2.1 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgx/v5 v5.5.5 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/driver/mysql v1.4.7 // indirect +) diff --git a/management/webserver/go.sum b/management/webserver/go.sum new file mode 100644 index 0000000..c1e3978 --- /dev/null +++ b/management/webserver/go.sum @@ -0,0 +1,223 @@ +chaitin.cn/dev/go/errors v0.0.0-20200717101723-df6132d53dc8/go.mod h1:u+ZD0shdyUt0UG9XfCgD1R5mSqXpTVoO5rwQXQeo3Eo= +chaitin.cn/dev/go/errors v0.0.0-20210324055134-dc5247602af6 h1:1Qa9ABk907/9ZrOLbbRcS8Fqq9VhjAF/mLjbSP1qAJY= +chaitin.cn/dev/go/errors v0.0.0-20210324055134-dc5247602af6/go.mod h1:u+ZD0shdyUt0UG9XfCgD1R5mSqXpTVoO5rwQXQeo3Eo= +chaitin.cn/dev/go/log v0.0.0-20221220104336-05125760b10c h1:Xn9IYkxmnpDcEpV+7JIR5ufEIexd1dhqKwpOLG1mYOE= +chaitin.cn/dev/go/log v0.0.0-20221220104336-05125760b10c/go.mod h1:xJIYwUoA2TX5mNg/RBrEPyE251BPwj+70/mM7UIhoxg= +chaitin.cn/dev/go/settings v0.0.0-20221220104336-05125760b10c h1:tXsraF7o9iUsQY6IwpDJusc6OFhB7iv/bBTfgR3MPUU= +chaitin.cn/dev/go/settings v0.0.0-20221220104336-05125760b10c/go.mod h1:fUvtmpG8Z8Zf5aciadL9a/vn5SB3knG7pdNJixDplPg= +github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= +github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.8.0 h1:ea0Xadu+sHlu7x5O3gKhRpQ1IKiMrSiHttPF0ybECuA= +github.com/bytedance/sonic v1.8.0/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gin-contrib/sessions v0.0.5 h1:CATtfHmLMQrMNpJRgzjWXD7worTh7g7ritsQfmF+0jE= +github.com/gin-contrib/sessions v0.0.5/go.mod h1:vYAuaUPqie3WUSsft6HUlCjlwwoJQs97miaG2+7neKY= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.9.0 h1:OjyFBKICoexlu99ctXNR2gg+c5pKrKMuyjgARg9qeY8= +github.com/gin-gonic/gin v1.9.0/go.mod h1:W1Me9+hsUSyj3CePGrd1/QrKJMSJ1Tu/0hFEH89961k= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= +github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= +github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= +github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.3.0 h1:/NQi8KHMpKWHInxXesC8yD4DhkXPrVhmnwYkjp9AmBA= +github.com/jackc/pgx/v5 v5.3.0/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8= +github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= +github.com/microsoft/go-mssqldb v0.17.0 h1:Fto83dMZPnYv1Zwx5vHHxpNraeEaUlQ/hhHLgZiaenE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pquerna/otp v1.4.0 h1:wZvl1TIVxKRThZIBiwOOHOGP/1+nZyWBil9Y2XNEDzg= +github.com/pquerna/otp v1.4.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.9 h1:rmenucSohSTiyL09Y+l2OCk+FrMxGMzho2+tjr5ticU= +github.com/ugorji/go/codec v1.2.9/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/datatypes v1.1.1 h1:XAjO7NNfUKVUvnS3+BkqMrPXxCAcxDlpOYbjnizxNCw= +gorm.io/datatypes v1.1.1/go.mod h1:u8GEgFjJ+GpsGfgHmBUcQqHm/937t3sj/SO9dvbndTg= +gorm.io/driver/mysql v1.4.7 h1:rY46lkCspzGHn7+IYsNpSfEv9tA+SU4SkkB+GFX125Y= +gorm.io/driver/mysql v1.4.7/go.mod h1:SxzItlnT1cb6e1e4ZRpgJN2VYtcqJgqnHxWr4wsP8oc= +gorm.io/driver/postgres v1.5.0 h1:u2FXTy14l45qc3UeCJ7QaAXZmZfDDv0YrthvmRq1l0U= +gorm.io/driver/postgres v1.5.0/go.mod h1:FUZXzO+5Uqg5zzwzv4KK49R8lvGIyscBOqYrtI1Ce9A= +gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= +gorm.io/driver/sqlite v1.4.3 h1:HBBcZSDnWi5BW3B3rwvVTc510KGkBkexlOg0QrmLUuU= +gorm.io/driver/sqlserver v1.4.1 h1:t4r4r6Jam5E6ejqP7N82qAJIJAht27EGT41HyPfXRw0= +gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/gorm v1.24.7-0.20230306060331-85eaf9eeda11 h1:9qNbmu21nNThCNnF5i2R3kw2aL27U8ZwbzccNjOmW0g= +gorm.io/gorm v1.24.7-0.20230306060331-85eaf9eeda11/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= +gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/management/webserver/main.go b/management/webserver/main.go new file mode 100644 index 0000000..4d55af5 --- /dev/null +++ b/management/webserver/main.go @@ -0,0 +1,225 @@ +package main + +import ( + "flag" + "fmt" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/gin-contrib/sessions" + "github.com/gin-contrib/sessions/cookie" + "github.com/gin-gonic/gin" + + "chaitin.cn/patronus/safeline-2/management/webserver/api" + "chaitin.cn/patronus/safeline-2/management/webserver/cmd" + "chaitin.cn/patronus/safeline-2/management/webserver/middleware" + "chaitin.cn/patronus/safeline-2/management/webserver/model" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/cron" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/fvm" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/log" + "chaitin.cn/patronus/safeline-2/management/webserver/rpc" +) + +var ( + logger = log.GetLogger("main") + + version = "undefined" + githash = "undefined" + buildstamp = "undefined" + goVersion = "undefined" +) + +func init() { + // do something that do not raise error +} + +func main() { + log.SetLogFormatter() + + fs := flag.NewFlagSet("webserver", flag.ExitOnError) + showVersion := fs.Bool("v", false, "show version") + cfgFile := fs.String("c", constants.ConfigFilePath, "config file path") + genCerts := fs.Bool("gen_certs", false, "generate certs") + showFSL := fs.Bool("show_fsl", false, "show full selectors") + push_fsl := fs.Bool("push_fsl", false, "compile and push fsl") + fakeLogs := fs.Bool("fake_logs", false, "fake logs") + resetUsername := fs.String("reset_user", "", "reset user") + if err := fs.Parse(os.Args[1:]); err != nil { + logger.Fatalln("Failed to parse args: ", err) + } + + if *showVersion { + i, _ := strconv.Atoi(buildstamp) + t := time.Unix(int64(i), 0).Format("2006-01-02 15:04:05") + fmt.Println("Version: ", version) + fmt.Println("Githash: ", githash) + fmt.Println("Build: ", t) + fmt.Println("Go version: ", goVersion) + return + } + + constants.Version = version + // init configs + if err := config.InitConfigs(*cfgFile); err != nil { + logger.Fatalln("Failed to init configs: ", err) + } + + if err := log.InitLogger(); err != nil { + logger.Fatalln("Failed to init db: ", err) + } + + if *genCerts { + if err := cmd.GenCerts(); err != nil { + logger.Fatalln("Failed to generate certs: ", err) + } + return + } + + logger.Info("Init database") + if err := database.InitDB(); err != nil { + logger.Fatalln("Failed to init db: ", err) + } + + if *showFSL { + if fullFSL, err := cmd.ShowFSL(); err != nil { + logger.Fatalln("Failed to generate fsl: ", err) + } else { + logger.Info(strings.ReplaceAll(strings.ReplaceAll(fullFSL, ";", ";\n"), "CREATE", "\nCREATE")) + } + return + } + + logger.Info("Init models") + if err := model.InitModels(); err != nil { + logger.Fatalln("Failed to init models: ", err) + } + + if len(*resetUsername) > 0 { + logger.Infoln("reset user:", *resetUsername) + cmd.ResetUser(*resetUsername) + logger.Infoln("success!") + return + } + + if *fakeLogs { + logger.Infoln("faking logs...") + cmd.FakeLogs() + logger.Infoln("success!") + return + } + + if *push_fsl { + logger.Infoln("push fsl...") + if err := cmd.PushFSL(); err != nil { + logger.Fatalln("Failed to generate fsl: ", err) + } + logger.Infoln("success!") + return + } + + logger.Info("Init FVM bytecode") + fvm.InitFVMBytecode() + + if err := cron.StartCron(); err != nil { + logger.Fatalln("Failed to start cron: ", err) + } + + if err := rpc.StartGRPCSever(); err != nil { + logger.Fatalln("Failed to start gRPC server: ", err) + } + + gin.SetMode(gin.ReleaseMode) + r := gin.Default() + + var option model.Options + database.GetDB().Where(&model.Options{Key: constants.SecretKey}).First(&option) + logger.Debugf("Secret: %s", option.Value) + store := cookie.NewStore([]byte(option.Value)) + r.Use(sessions.Sessions("session", store)) + + publicRouters := r.Group("/api") + publicRouters.POST(api.Login, api.PostLogin) + publicRouters.POST(api.Logout, api.PostLogout) + publicRouters.POST(api.Behaviour, api.PostBehaviour) + publicRouters.POST(api.FalsePositives, api.PostFalsePositives) + publicRouters.GET(api.OTPUrl, api.GetOTPUrl) + publicRouters.GET(api.Version, api.GetVersion) + publicRouters.GET(api.UpgradeTips, api.GetUpgradeTips) + // test use + publicRouters.GET("/Ping", func(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{ + "message": "pong", + }) + }) + + limitedRouters := r.Group("/api") + noAuth, existed := os.LookupEnv("NO_AUTH") + if existed && len(noAuth) >= 0 { + logger.Warn("No auth") + } else { + limitedRouters.Use(middleware.AuthRequired) + } + readOnly, existed := os.LookupEnv("READ_ONLY") + if existed && len(readOnly) >= 0 { + logger.Warn("Read only") + limitedRouters.Use(middleware.ReadOnly) + } + + limitedRouters.GET(api.User, api.GetUser) + limitedRouters.GET(api.CurrentUser, api.GetCurrentUser) + + // Admin-only user management + adminRouters := limitedRouters.Group("") + adminRouters.Use(middleware.RequireRole("admin")) + adminRouters.GET(api.Users, api.GetUsers) + adminRouters.POST(api.Users, api.PostUser) + adminRouters.PUT(api.Users+"/:id", api.PutUser) + adminRouters.DELETE(api.Users+"/:id", api.DeleteUser) + adminRouters.POST(api.Users+"/:id/reset-password", api.PostResetPassword) + + // Operator and above (read + policy management) + operatorRouters := limitedRouters.Group("") + operatorRouters.Use(middleware.RequireRole("admin", "operator")) + + operatorRouters.GET(api.DetectLogList, api.GetDetectLogList) + operatorRouters.GET(api.DetectLogDetail, api.GetDetectLogDetail) + + operatorRouters.POST(api.Website, api.PostWebsite) + operatorRouters.PUT(api.Website, api.PutWebsite) + operatorRouters.DELETE(api.Website, api.DeleteWebsite) + operatorRouters.GET(api.Website, api.GetWebsite) + + operatorRouters.POST(api.UploadSSLCert, api.PostUploadSSLCert) + operatorRouters.POST(api.SSLCert, api.PostSSLCert) + + operatorRouters.POST(api.PolicyRule, api.PostPolicyRule) + operatorRouters.PUT(api.PolicyRule, api.PutPolicyRule) + operatorRouters.DELETE(api.PolicyRule, api.DeletePolicyRule) + operatorRouters.GET(api.PolicyRule, api.GetPolicyRule) + operatorRouters.PUT(api.SwitchPolicyRule, api.PutSwitchPolicyRule) + + // All authenticated users can access dashboard and view policies + limitedRouters.GET(api.DashboardCounts, api.GetDashboardCounts) + limitedRouters.GET(api.DashboardSites, api.GetDashboardSites) + limitedRouters.GET(api.DashboardQps, api.GetDashboardQps) + limitedRouters.GET(api.DashboardRequests, api.GetDashboardRequests) + limitedRouters.GET(api.DashboardIntercepts, api.GetDashboardIntercepts) + + limitedRouters.GET(api.PolicyGroupGlobal, api.GetPolicyGroupGlobal) + limitedRouters.PUT(api.PolicyGroupGlobal, api.PutPolicyGroupGlobal) + limitedRouters.POST(api.PolicyGroupPreset, api.PostPolicyGroupPreset) + + limitedRouters.GET(api.SrcIPConfig, api.GetSrcIPConfig) + limitedRouters.PUT(api.SrcIPConfig, api.PutSrcIPConfig) + + logger.Info("Staring...") + if err := r.Run(config.GlobalConfig.Server.ListenAddr); err != nil { + logger.Fatalln("Error occurred when running web server: ", err) + } +} diff --git a/management/webserver/middleware/auth.go b/management/webserver/middleware/auth.go new file mode 100644 index 0000000..724b599 --- /dev/null +++ b/management/webserver/middleware/auth.go @@ -0,0 +1,93 @@ +package middleware + +import ( + "net/http" + + "github.com/gin-contrib/sessions" + "github.com/gin-gonic/gin" + + "chaitin.cn/patronus/safeline-2/management/webserver/api/response" + "chaitin.cn/patronus/safeline-2/management/webserver/model" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" +) + +func AuthRequired(c *gin.Context) { + session := sessions.Default(c) + + user := session.Get(constants.DefaultSessionUserKey) + if user == nil { + response.Error(c, response.ErrorLoginRequired, http.StatusUnauthorized) + c.Abort() + return + } + + // Load user from database to get current role/status + db := database.GetDB() + var userModel model.User + res := db.First(&userModel, user) + if res.RowsAffected == 0 { + response.Error(c, response.ErrorLoginRequired, http.StatusUnauthorized) + c.Abort() + return + } + + // Check if user is still enabled + if !userModel.IsEnabled { + session.Clear() + session.Save() + response.Error(c, response.JSONBody{Err: "account_disabled", Msg: "Account has been disabled"}, http.StatusForbidden) + c.Abort() + return + } + + // Store user info in context for downstream handlers + c.Set("user_id", int(userModel.ID)) + c.Set("user_role", userModel.Role) + c.Set("username", userModel.Username) + + // extend session expired time + session.Options(sessions.Options{ + Path: "/", + MaxAge: 3600 * 24 * 7, + }) + + if err := session.Save(); err != nil { + response.Error(c, response.JSONBody{Err: response.ErrInternalError, Msg: "Error occurred when creating sessions"}, http.StatusInternalServerError) + return + } + + c.Next() +} + +// RequireRole returns a middleware that checks if the user has one of the required roles. +func RequireRole(roles ...string) gin.HandlerFunc { + roleSet := make(map[string]bool) + for _, r := range roles { + roleSet[r] = true + } + return func(c *gin.Context) { + userRole, exists := c.Get("user_role") + if !exists { + response.Error(c, response.ErrorLoginRequired, http.StatusUnauthorized) + c.Abort() + return + } + + role, ok := userRole.(string) + if !ok || !roleSet[role] { + response.Error(c, response.JSONBody{Err: "forbidden", Msg: "Insufficient permissions"}, http.StatusForbidden) + c.Abort() + return + } + + c.Next() + } +} + +func ReadOnly(c *gin.Context) { + if c.Request.Method != "GET" && c.Request.Method != "HEAD" && c.Request.Method != "OPTIONS" { + response.Error(c, response.ErrorReadOnly, http.StatusBadRequest) + c.Abort() + } +} diff --git a/management/webserver/model/base.go b/management/webserver/model/base.go new file mode 100644 index 0000000..d8e516a --- /dev/null +++ b/management/webserver/model/base.go @@ -0,0 +1,16 @@ +package model + +import ( + "time" + + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/log" +) + +// Base is a replacement for gorm.Model without DeletedAt, which is considered to be not good. +type Base struct { + ID uint `gorm:"primarykey" json:"id"` + CreatedAt time.Time ` json:"created_at"` + UpdatedAt time.Time ` json:"updated_at"` +} + +var logger = log.GetLogger("model") diff --git a/management/webserver/model/behaviour.go b/management/webserver/model/behaviour.go new file mode 100644 index 0000000..b644fd9 --- /dev/null +++ b/management/webserver/model/behaviour.go @@ -0,0 +1,7 @@ +package model + +type Behaviour struct { + Base + SrcRouter string `json:"src_router"` + DstRouter string `json:"dst_router"` +} diff --git a/management/webserver/model/db_patch_1_4_0.go b/management/webserver/model/db_patch_1_4_0.go new file mode 100644 index 0000000..20235e9 --- /dev/null +++ b/management/webserver/model/db_patch_1_4_0.go @@ -0,0 +1,38 @@ +package model + +import ( + "strings" + + "chaitin.cn/dev/go/errors" + "gorm.io/gorm" +) + +type sqlResult struct { + Ids string `json:"ids"` +} + +func DBPatch140(tx *gorm.DB) error { + if !tx.Migrator().HasTable(&SystemStatistics{}) { + return nil + } + + var result []sqlResult + //SELECT string_agg(id::text, ',') as ids FROM mgt_system_statistics GROUP BY (type, website, created_at) HAVING COUNT(*) > 1) as tmp + res := tx.Model(&SystemStatistics{}).Select("string_agg(id::text, ',') as ids").Group("type, website, created_at").Having("COUNT(*) > 1").Find(&result) + if res.Error != nil { + return errors.Wrap(res.Error, "Failed to select data") + } + + if len(result) <= 0 { + return nil + } + + var deleteIds []string + for _, s := range result { + tmpIds := strings.Split(s.Ids, ",") + deleteIds = append(deleteIds, tmpIds...) + } + deleteRes := tx.Delete(&SystemStatistics{}, deleteIds) + + return errors.Wrap(deleteRes.Error, "Failed to delete") +} diff --git a/management/webserver/model/detectlog.go b/management/webserver/model/detectlog.go new file mode 100644 index 0000000..b12473a --- /dev/null +++ b/management/webserver/model/detectlog.go @@ -0,0 +1,196 @@ +package model + +import ( + "fmt" + "math/rand" + "strings" + "time" + + "chaitin.cn/dev/go/errors" + + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" + "chaitin.cn/patronus/safeline-2/management/webserver/utils" +) + +// DetectLog is designed to be used in response, not a good naming. +type DetectLog struct { + DetectLogBasic + DetectLogDetail + EventId string `json:"event_id"` // to eliminate ambiguous + Website string `json:"website"` + AttackType string `json:"attack_type"` + Module string `json:"module"` + Reason string `json:"reason"` +} + +func getRuleModule(ruleId string, attackType int) string { + var module string + if strings.HasPrefix(ruleId, "m_rule") { // `m_rule/65543` + module = "m_rule" + } else if strings.HasPrefix(ruleId, "/") { + if attackType == -2 { + module = "whitelist" + } else { // if attackType == -3 + module = "blacklist" + } + } else { + module = ruleId + } + return constants.RuleModule[module] +} + +func getRuleReason(ruleId string, attackType int) string { + var reason string + if strings.HasPrefix(ruleId, "m_rule") { // `m_rule/65543` + reason = constants.RuleReason[strings.ReplaceAll(ruleId, "m_rule/", "")] + } else if strings.HasPrefix(ruleId, "/") { + reason = ruleId[1:] + } else { + reason = fmt.Sprintf("检测到 %s 攻击", getAttackType(attackType)) + } + return reason +} + +func getAttackType(at int) string { + atStr, ok := constants.AttackType[at] + if !ok { + atStr = constants.AttackType[62] // unknown + } + return atStr +} + +func getCountry(country string) string { + countryStr, ok := constants.CountryCode[country] + if !ok { + countryStr = "" + } + return countryStr +} + +func TransformDetectLog(basic *DetectLogBasic, detail *DetectLogDetail) (*DetectLog, error) { + if basic == nil { + return nil, errors.New("basic *DetectLogBasic cannot be nil") + } + if detail != nil && basic.EventId != detail.EventId { + return nil, errors.New("EventId field should be the same for basic and detail") + } + + basic.Country = getCountry(basic.Country) + dLog := DetectLog{ + DetectLogBasic: *basic, + EventId: basic.EventId, + Website: utils.BuildUrl(basic.Protocol, basic.Host, basic.DstPort, basic.UrlPath), + AttackType: getAttackType(basic.AttackType), + Module: getRuleModule(basic.RuleId, basic.AttackType), + Reason: getRuleReason(basic.RuleId, basic.AttackType), + } + if detail != nil { + dLog.DetectLogDetail = *detail + } + + return &dLog, nil +} + +type DetectLogBasic struct { + ID uint `json:"id" gorm:"primarykey"` + EventId string `json:"event_id" gorm:"uniqueIndex;not null"` + + SiteUUID string `json:"site_uuid" gorm:"column:site_uuid"` + SrcIp string `json:"src_ip" gorm:"index"` + SocketIp string `json:"socket_ip" gorm:"index"` + + Protocol int `json:"protocol"` + Host string `json:"host"` + UrlPath string `json:"url_path"` + DstPort uint `json:"dst_port"` + + Country string `json:"country"` + Province string `json:"province"` + City string `json:"city"` + + AttackType int `json:"attack_type" gorm:"index"` + RiskLevel int `json:"risk_level" gorm:"index"` + Action int `json:"action" gorm:"index"` + RuleId string `json:"rule_id"` + + Timestamp int64 `json:"timestamp" gorm:"index"` +} + +type DetectLogDetail struct { + ID uint ` gorm:"primarykey"` + EventId string ` gorm:"uniqueIndex;not null"` + + SrcPort uint `json:"src_port"` + DstIp string `json:"dst_ip"` + + Method string `json:"method"` + QueryString string `json:"query_string"` + StatusCode uint `json:"status_code"` + + ReqHeader string `json:"req_header"` + ReqBody string `json:"req_body"` + RspHeader string `json:"rsp_header"` + RspBody string `json:"rsp_body"` + + Payload string `json:"payload"` + Location string `json:"location"` + DecodePath string `json:"decode_path"` +} + +func InitDetectLogSamples() { + db := database.GetDB() + var detectLogBasicList []DetectLogBasic + var detectLogDetailList []DetectLogDetail + + timestamp := time.Now().Unix() + detail := DetectLogDetail{ + SrcPort: 58694, + DstIp: "10.2.35.143", + Method: "GET", + QueryString: "", + ReqHeader: "GET /webshell.php HTTP/1.1\nUser-Agent: curl/7.77.0\nAccept: */*\n\n\"", + ReqBody: "", + RspHeader: "", + RspBody: "", + Payload: "", + Location: "urlpath", + DecodePath: "", + } + + protocolList := []int{constants.ProtocolHTTP, constants.ProtocolHTTPS} + portList := []uint{80, 443} + provinceList := []string{} + cityList := []string{} + ipList := []string{} + ruleIdList := []string{} + + for i := 0; i < 100; i++ { + randInt := rand.Intn(1000) + eventId := utils.RandStr(32) + + basic := DetectLogBasic{} + basic.EventId = eventId + basic.SrcIp = ipList[randInt%len(ipList)] + basic.SocketIp = ipList[randInt%len(ipList)] + basic.Protocol = protocolList[randInt%len(protocolList)] + basic.DstPort = portList[randInt%len(portList)] + basic.Host = fmt.Sprintf("%s.com", utils.RandStr(5)) + basic.UrlPath = fmt.Sprintf("/%s", utils.RandStr(10)) + basic.Country = "CN" + basic.Province = provinceList[randInt%len(provinceList)] + basic.City = cityList[randInt%len(cityList)] + basic.AttackType = randInt % 32 + basic.RiskLevel = randInt % 4 + basic.Action = randInt % 2 + basic.RuleId = ruleIdList[randInt%len(ruleIdList)] + basic.Timestamp = timestamp - int64(randInt*100) + + detailCopy := detail + detailCopy.EventId = eventId + detectLogBasicList = append(detectLogBasicList, basic) + detectLogDetailList = append(detectLogDetailList, detailCopy) + } + db.CreateInBatches(detectLogBasicList, 100) + db.CreateInBatches(detectLogDetailList, 100) +} diff --git a/management/webserver/model/init.go b/management/webserver/model/init.go new file mode 100644 index 0000000..1a2ef99 --- /dev/null +++ b/management/webserver/model/init.go @@ -0,0 +1,45 @@ +package model + +import ( + "gorm.io/gorm" + + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" +) + +func InitModels() error { + db := database.GetDB() + err := db.Transaction(func(tx *gorm.DB) error { + // + if err := DBPatch140(tx); err != nil { + return err + } + return nil + }) + if err != nil { + return err + } + + if err := db.AutoMigrate(&User{}, &DetectLogBasic{}, &DetectLogDetail{}, &Behaviour{}, &Options{}, &Website{}, &PolicyRule{}, &SystemStatistics{}); err != nil { + return err + } + + if err := initAdminUser(); err != nil { + return err + } + + if err := initOptions(); err != nil { + return err + } + + if err := initPolicyGroupGlobal(); err != nil { + return err + } + + if err := initSrcIPConfig(); err != nil { + return err + } + + //InitDetectLogSamples() + + return nil +} diff --git a/management/webserver/model/option.go b/management/webserver/model/option.go new file mode 100644 index 0000000..b7ad429 --- /dev/null +++ b/management/webserver/model/option.go @@ -0,0 +1,27 @@ +package model + +import ( + "gorm.io/gorm/clause" + + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" + "chaitin.cn/patronus/safeline-2/management/webserver/utils" +) + +type Options struct { + Base + Key string `gorm:"column:key;uniqueIndex"` + Value string `gorm:"column:value;"` +} + +func initOptions() error { + db := database.GetDB() + + secretKey := Options{Key: constants.SecretKey, Value: utils.RandStr(32)} + db.Clauses(clause.OnConflict{DoNothing: true}).Create(&secretKey) + + machineId := Options{Key: constants.MachineID, Value: utils.RandStr(32)} + _ = db.Clauses(clause.OnConflict{DoNothing: true}).Create(&machineId) + + return nil +} diff --git a/management/webserver/model/policygroup.go b/management/webserver/model/policygroup.go new file mode 100644 index 0000000..d63c530 --- /dev/null +++ b/management/webserver/model/policygroup.go @@ -0,0 +1,216 @@ +package model + +import ( + "encoding/json" + + "gorm.io/gorm" + "gorm.io/gorm/clause" + + "chaitin.cn/dev/go/errors" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" +) + +type PolicyGroup map[string]string + +const ( + StrictMode = "strict" + DefaultMode = "default" + DisableMode = "disable" + + SocketIP = "socket_ip" + HTTPHeader = "http_header" +) + +type SrcIPConfig struct { + Source string `json:"source"` // socket_ip, http_header + Value string `json:"value"` +} + +type SkynetConfigModule struct { + DetectConfig interface{} `json:"detect_config,omitempty"` + HighRiskAction string `json:"high_risk_action"` + MediumRiskAction string `json:"medium_risk_action"` + LowRiskAction string `json:"low_risk_action"` + HighRiskEnableLog int `json:"high_risk_enable_log"` + MediumRiskEnableLog int `json:"medium_risk_enable_log"` + LowRiskEnableLog int `json:"low_risk_enable_log"` + State string `json:"state"` +} + +type SkynetConfig struct { + DetectConfig string `json:"detect_config"` + DeepDetect bool `json:"deep_detect"` + Timeout interface{} `json:"timeout"` + + Modules map[string]*SkynetConfigModule `json:"modules"` +} + +func GetSrcIPConfig(db *gorm.DB) (*SrcIPConfig, error) { + var scOption Options + db.Where(&Options{Key: constants.SrcIPConfig}).First(&scOption) + + var sc SrcIPConfig + err := json.Unmarshal([]byte(scOption.Value), &sc) + if err != nil { + return nil, err + } + + return &sc, nil +} + +func initSrcIPConfig() error { + db := database.GetDB() + + // policyGroupGlobal config, three mode: strict/default/disable + var srcIPConfig = SrcIPConfig{ + Source: SocketIP, + Value: "", + } + + scStr, err := json.Marshal(srcIPConfig) + if err != nil { + return err + } + + pgg := Options{Key: constants.SrcIPConfig, Value: string(scStr)} + db.Clauses(clause.OnConflict{DoNothing: true}).Create(&pgg) + + return nil +} + +func GetSkynetConfig(db *gorm.DB) (string, error) { + // skynetConfigStr skynet config in 5.3.9 + const skynetConfigStr = "{\"decode_config\":{\"decode_methods\":[\"url decode\",\"JSON\",\"base64\",\"hex\",\"eval\",\"XML\",\"PHP deserialize\",\"utf7\"]},\"deep_detect\":false,\"modules\":{\"m_asp_code_injection\":{\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_cmd_injection\":{\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_csrf\":{\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_file_include\":{\"detect_config\":{\"detect_suspicious_schema\":true},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_file_upload\":{\"detect_config\":{\"action_of_handling_custome_file\":[\"deny transmission\"],\"detect_file_content\":true,\"detect_real_file_content\":false,\"enable_module_in_detecting_file_content\":[\"java\",\"asp_code_injection\",\"php_code_injection\"],\"file_type_of_custome_action\":[],\"forbidden_extra_token\":false,\"forbidden_multiple_filename\":true,\"forbidden_suspicious_filename\":true},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_http\":{\"detect_config\":{\"warning_http_parse_failed\":false,\"warning_suspicious_http_version\":false},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"disabled\"},\"m_java\":{\"detect_config\":{\"detect_lookup\":false},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_java_unserialize\":{\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_php_code_injection\":{\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_php_unserialize\":{\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_response\":{\"detect_config\":{\"detection_configure\":[\"detect_jsp_code_leak\",\"detect_php_code_leak\",\"detect_webshell\"],\"error_types\":[\"directory indexing\",\"SQL execution error\",\"server exception\"]},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"disabled\"},\"m_rule\":{\"detect_config\":{\"check_info_leak_by_rsp\":false,\"compatibility_mode\":false,\"detection_configure\":[\"detect_phpinfo\",\"detect_admin_page\",\"detect_backdoor\",\"detect_php_vuln\",\"detect_nginx\",\"detect_dedecms\",\"detect_apache\",\"detect_xml\",\"detect_struts2\",\"detect_java_vuln\",\"detect_php168\",\"detect_wordpress\",\"detect_directory_traversal\",\"detect_iis\",\"detect_gogs_gitea\",\"detect_thinkphp\",\"detect_jenkins\",\"detect_ecshop\",\"detect_nexus\",\"detect_drupal\",\"detect_ghostscript\",\"detect_atlassian\",\"detect_weblogic\",\"detect_coremail\",\"detect_phpcms\",\"detect_spring\",\"detect_southidc\",\"detect_fastjson\",\"detect_tbk_dvr\",\"detect_joomla\",\"detect_ecology_oa\",\"detect_jackson\",\"detect_xstream\",\"detect_activemq\",\"detect_solr\",\"detect_csii\",\"detect_big_ip\",\"detect_apisix\",\"detect_druid\",\"detect_log4j\"],\"info_leak_types\":[\"test file\",\"backup file\",\"code repository\",\"server sensitive file\"],\"rules_config\":{\"disable_ruleid\":[],\"enable_ruleid\":[],\"rules_status\":[]}},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_scanner\":{\"detect_config\":{\"language_types\":[\"python_scanner\",\"go_scanner\"],\"other_types\":[\"normal_scanner\"],\"spider_types\":[]},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"disabled\"},\"m_sqli\":{\"detect_config\":{\"detect_non_injection_sql\":true},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_ssrf\":{\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_ssti\":{\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_xss\":{\"detect_config\":{\"detect_complete_html\":true},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"}},\"timeout\":{\"threshold\":1000,\"log\":\"enabled\"}}" + var sc SkynetConfig + err := json.Unmarshal([]byte(skynetConfigStr), &sc) + if err != nil { + return "", err + } + + var pggOption Options + db.Where(&Options{Key: constants.PolicyGroupGlobal}).First(&pggOption) + + var pgg PolicyGroup + err = json.Unmarshal([]byte(pggOption.Value), &pgg) + if err != nil { + return "", err + } + + for module, mode := range pgg { + switch mode { + case StrictMode: + scm := sc.Modules[module] + scm.HighRiskAction = "deny" + scm.MediumRiskAction = "deny" + scm.LowRiskAction = "deny" + scm.State = "enabled" + // Enable aggressive detection options for strict mode + if scm.DetectConfig == nil { + scm.DetectConfig = map[string]interface{}{} + } + dcfg := scm.DetectConfig.(map[string]interface{}) + switch module { + case "m_rule": + dcfg["check_info_leak_by_rsp"] = true + dcfg["compatibility_mode"] = false + case "m_java": + dcfg["detect_lookup"] = true + case "m_file_upload": + dcfg["detect_real_file_content"] = true + dcfg["detect_file_content"] = true + dcfg["forbidden_extra_token"] = true + dcfg["forbidden_multiple_filename"] = true + dcfg["forbidden_suspicious_filename"] = true + case "m_file_include": + dcfg["detect_suspicious_schema"] = true + case "m_sqli": + dcfg["detect_non_injection_sql"] = true + case "m_xss": + dcfg["detect_complete_html"] = true + case "m_http": + dcfg["warning_http_parse_failed"] = true + dcfg["warning_suspicious_http_version"] = true + case "m_response": + dcfg["detection_configure"] = []string{ + "detect_jsp_code_leak", "detect_php_code_leak", + "detect_webshell", "detect_asp_code_leak", + } + dcfg["error_types"] = []string{ + "directory indexing", "SQL execution error", + "server exception", "stack trace leak", + } + case "m_scanner": + dcfg["language_types"] = []string{ + "python_scanner", "go_scanner", "java_scanner", + "php_scanner", "ruby_scanner", "perl_scanner", + } + dcfg["other_types"] = []string{ + "normal_scanner", "sqlmap", "nikto", + "nmap", "dirbuster", "gobuster", + } + dcfg["spider_types"] = []string{"normal_spider"} + } + scm.DetectConfig = dcfg + sc.Modules[module] = scm + case DefaultMode: + scm := sc.Modules[module] + scm.HighRiskAction = "deny" + scm.MediumRiskAction = "continue" + scm.LowRiskAction = "continue" + scm.State = "enabled" + sc.Modules[module] = scm + case DisableMode: + scm := sc.Modules[module] + scm.HighRiskAction = "continue" + scm.MediumRiskAction = "continue" + scm.LowRiskAction = "continue" + scm.State = "disabled" + sc.Modules[module] = scm + default: + return "", errors.New("no such mode") + } + } + + scStr, err := json.Marshal(sc) + if err != nil { + return "", err + } + + return string(scStr), nil +} + +func initPolicyGroupGlobal() error { + db := database.GetDB() + + // policyGroupGlobal config, three mode: strict/default/disable + var policyGroupGlobal = PolicyGroup{ + "m_asp_code_injection": DefaultMode, + "m_cmd_injection": DefaultMode, + "m_csrf": DefaultMode, + "m_file_include": DefaultMode, + "m_file_upload": DefaultMode, + "m_http": DefaultMode, + "m_java": DefaultMode, + "m_java_unserialize": DefaultMode, + "m_php_code_injection": DefaultMode, + "m_php_unserialize": DefaultMode, + "m_response": DefaultMode, + "m_rule": DefaultMode, + "m_scanner": DefaultMode, + "m_sqli": DefaultMode, + "m_ssrf": DefaultMode, + "m_ssti": DefaultMode, + "m_xss": DefaultMode, + } + + pggStr, err := json.Marshal(policyGroupGlobal) + if err != nil { + return err + } + + pgg := Options{Key: constants.PolicyGroupGlobal, Value: string(pggStr)} + db.Clauses(clause.OnConflict{DoNothing: true}).Create(&pgg) + + return nil +} diff --git a/management/webserver/model/policyrule.go b/management/webserver/model/policyrule.go new file mode 100644 index 0000000..0dc9166 --- /dev/null +++ b/management/webserver/model/policyrule.go @@ -0,0 +1,30 @@ +package model + +import "gorm.io/datatypes" + +type PolicyRule struct { + Base + Action int `gorm:"action" json:"action"` + Comment string `gorm:"comment" json:"comment"` + Pattern datatypes.JSON `gorm:"pattern" json:"pattern"` + IsEnabled bool `gorm:"is_enabled;default=true" json:"is_enabled"` +} + +type PolicyRulePattern struct { + K string `json:"k"` + Op string `json:"op"` + V string `json:"v"` +} + +const ( + KeySrcIP = "src_ip" + KeyURI = "uri" + KeyHost = "host" + + OpEq = "eq" // 完全相等 + OpMatch = "match" // 模糊匹配 + OpCIDR = "cidr" // CIDR + OpHas = "has" // 关键字 + OpPrefix = "prefix" // 前缀关键字 + OpRe = "re" // 正则 +) diff --git a/management/webserver/model/statistics.go b/management/webserver/model/statistics.go new file mode 100644 index 0000000..0a731cc --- /dev/null +++ b/management/webserver/model/statistics.go @@ -0,0 +1,11 @@ +package model + +import "time" + +type SystemStatistics struct { + ID uint `json:"id" gorm:"primarykey"` + Type string `json:"type" gorm:"index;uniqueIndex:type_website_createdat"` + Value int64 `json:"value"` + CreatedAt time.Time `json:"created_at" gorm:"index;uniqueIndex:type_website_createdat"` + Website string `json:"website" gorm:"index;uniqueIndex:type_website_createdat"` +} diff --git a/management/webserver/model/user.go b/management/webserver/model/user.go new file mode 100644 index 0000000..f641eb6 --- /dev/null +++ b/management/webserver/model/user.go @@ -0,0 +1,118 @@ +package model + +import ( + "crypto/rand" + "math/big" + "strings" + + "golang.org/x/crypto/bcrypt" + "gorm.io/gorm/clause" + + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" +) + +type User struct { + Base + Username string `gorm:"uniqueIndex;not null"` + Password string `gorm:"-"` // transient, not stored + PasswordHash string + Comment string + Role string `gorm:"default:viewer"` // admin, operator, viewer + + TFAEnabled bool `gorm:"column:tfa_enabled;default:true"` + TFASecret string `gorm:"column:tfa_secret"` + LastLoginTime int64 `gorm:"default:0"` + IsEnabled bool `gorm:"default:true"` +} + +// HasPermission checks if the user role has the required permission level. +// admin > operator > viewer +func (u *User) HasPermission(requiredRole string) bool { + roleHierarchy := map[string]int{ + "viewer": 1, + "operator": 2, + "admin": 3, + } + userLevel, ok := roleHierarchy[u.Role] + if !ok { + return false + } + requiredLevel, ok := roleHierarchy[requiredRole] + if !ok { + return false + } + return userLevel >= requiredLevel +} + +// CheckPassword verifies a plaintext password against the bcrypt hash. +func (u *User) CheckPassword(password string) bool { + if u.PasswordHash == "" { + return false + } + err := bcrypt.CompareHashAndPassword([]byte(u.PasswordHash), []byte(password)) + return err == nil +} + +// SetPassword hashes and sets the password. +func (u *User) SetPassword(password string) error { + hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + return err + } + u.PasswordHash = string(hash) + return nil +} + +// GenerateRandomPassword generates a random 16-char password. +func GenerateRandomPassword() (string, error) { + const chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*" + var sb strings.Builder + for i := 0; i < 16; i++ { + n, err := rand.Int(rand.Reader, big.NewInt(int64(len(chars)))) + if err != nil { + return "", err + } + sb.WriteByte(chars[n.Int64()]) + } + return sb.String(), nil +} + +func initAdminUser() error { + db := database.GetDB() + + // Check if admin already exists + var existing User + res := db.Where(&User{Username: constants.SuperUser}).First(&existing) + if res.RowsAffected > 0 { + return nil + } + + // Generate random password for admin + defaultPassword, err := GenerateRandomPassword() + if err != nil { + return err + } + + hash, err := bcrypt.GenerateFromPassword([]byte(defaultPassword), bcrypt.DefaultCost) + if err != nil { + return err + } + + admin := User{ + Username: constants.SuperUser, + PasswordHash: string(hash), + Role: "admin", + Comment: "Default admin user", + TFAEnabled: true, + IsEnabled: true, + } + + db.Clauses(clause.OnConflict{DoNothing: true}).Create(&admin) + + // Log the default password (will appear in container logs) + // In production, the mgt-cli reset-admin command handles this + _ = defaultPassword + + return nil +} diff --git a/management/webserver/model/website.go b/management/webserver/model/website.go new file mode 100644 index 0000000..1f39b25 --- /dev/null +++ b/management/webserver/model/website.go @@ -0,0 +1,25 @@ +package model + +import "gorm.io/datatypes" + +type Website struct { + Base + Comment string `gorm:"comment" json:"comment"` + ServerNames datatypes.JSON `gorm:"server_names" json:"server_names"` + Ports datatypes.JSON `gorm:"ports" json:"ports"` + Upstreams datatypes.JSON `gorm:"upstreams" json:"upstreams"` + + CertFilename string `gorm:"cert_filename" json:"cert_filename"` + KeyFilename string `gorm:"key_filename" json:"key_filename"` + + IsEnabled bool `gorm:"is_enabled;default=true" json:"is_enabled"` + + // Rate limiting (nginx limit_req) + RateLimitEnabled bool `gorm:"rate_limit_enabled;default:false" json:"rate_limit_enabled"` + RateLimitRPS int `gorm:"rate_limit_rps;default:100" json:"rate_limit_rps"` + RateLimitBurst int `gorm:"rate_limit_burst;default:200" json:"rate_limit_burst"` + RateLimitAction string `gorm:"rate_limit_action;default:reject" json:"rate_limit_action"` + + // JS Challenge (human verification) + ChallengeEnabled bool `gorm:"challenge_enabled;default:false" json:"challenge_enabled"` +} diff --git a/management/webserver/pkg/config/config.go b/management/webserver/pkg/config/config.go new file mode 100644 index 0000000..6e1c301 --- /dev/null +++ b/management/webserver/pkg/config/config.go @@ -0,0 +1,56 @@ +package config + +import ( + "os" + + "chaitin.cn/dev/go/settings" +) + +var ( + GlobalConfig = DefaultGlobalConfig() +) + +func InitConfigs(configFilePath string) error { + s, err := settings.New(configFilePath) + if err != nil { + return err + } + + if err = GlobalConfig.DB.Load(s); err != nil { + return err + } + + if err = GlobalConfig.Log.Load(s); err != nil { + return err + } + + if err = GlobalConfig.Server.Load(s); err != nil { + return err + } + + if err = GlobalConfig.Detector.Load(s); err != nil { + return err + } + + if err = GlobalConfig.Telemetry.Load(s); err != nil { + return err + } + + if err = GlobalConfig.GPRC.Load(s); err != nil { + return err + } + + if err := settings.Unmarshal("platform_addr", &GlobalConfig.PlatformAddr); err != nil { + return err + } + + if v, ok := os.LookupEnv("MANAGEMENT_RESOURCES_DIR"); ok { + GlobalConfig.MgtResDir = v + } + + if v, ok := os.LookupEnv("NGINX_RESOURCES_DIR"); ok { + GlobalConfig.NgxResDir = v + } + + return nil +} diff --git a/management/webserver/pkg/config/db.go b/management/webserver/pkg/config/db.go new file mode 100644 index 0000000..2a3bcab --- /dev/null +++ b/management/webserver/pkg/config/db.go @@ -0,0 +1,45 @@ +package config + +import ( + "net/url" + "os" + + "chaitin.cn/dev/go/settings" +) + +type DBConfig struct { + URL string `yaml:"url"` + LogSQL bool `yaml:"log_sql"` + SSLMode bool `yaml:"ssl_mode"` +} + +func DefaultDBConfig() DBConfig { + return DBConfig{ + URL: "postgres://safeline-ce:safeline-ce@127.0.0.1/safeline-ce", + LogSQL: false, + SSLMode: false, + } +} + +func (dbc *DBConfig) Load(setting *settings.Setting) error { + if err := setting.Unmarshal("db", dbc); err != nil { + return err + } + + if v, ok := os.LookupEnv("DATABASE_URL"); ok { + dbc.URL = v + } + + dbURL, err := url.Parse(dbc.URL) + if err != nil { + return err + } + q := dbURL.Query() + if !dbc.SSLMode { + q.Set("sslmode", "disable") + } + dbURL.RawQuery = q.Encode() + dbc.URL = dbURL.String() + + return nil +} diff --git a/management/webserver/pkg/config/detector.go b/management/webserver/pkg/config/detector.go new file mode 100644 index 0000000..1ba386b --- /dev/null +++ b/management/webserver/pkg/config/detector.go @@ -0,0 +1,25 @@ +package config + +import ( + "chaitin.cn/dev/go/settings" +) + +type DetectorConfig struct { + Addr string `yaml:"addr"` + FslBytecode string `yaml:"fsl_bytecode"` +} + +func DefaultDetectorConfig() DetectorConfig { + return DetectorConfig{ + Addr: "http://127.0.0.1:8001", + FslBytecode: "bytecode", + } +} + +func (d *DetectorConfig) Load(setting *settings.Setting) error { + if err := setting.Unmarshal("detector", d); err != nil { + return err + } + + return nil +} diff --git a/management/webserver/pkg/config/global.go b/management/webserver/pkg/config/global.go new file mode 100644 index 0000000..93de722 --- /dev/null +++ b/management/webserver/pkg/config/global.go @@ -0,0 +1,27 @@ +package config + +type Config struct { + Log LogConfig + DB DBConfig + Server ServerConfig + Detector DetectorConfig + Telemetry TelemetryConfig + GPRC GRPCConfig + PlatformAddr string + MgtResDir string + NgxResDir string +} + +func DefaultGlobalConfig() Config { + return Config{ + Log: DefaultLogConfig(), + DB: DefaultDBConfig(), + Server: DefaultServerConfig(), + Detector: DefaultDetectorConfig(), + Telemetry: DefaultTelemetryConfig(), + GPRC: DefaultGRPCConfig(), + PlatformAddr: "", + MgtResDir: "/resources/management", + NgxResDir: "/resources/nginx", + } +} diff --git a/management/webserver/pkg/config/grpc.go b/management/webserver/pkg/config/grpc.go new file mode 100644 index 0000000..294dc2c --- /dev/null +++ b/management/webserver/pkg/config/grpc.go @@ -0,0 +1,23 @@ +package config + +import ( + "chaitin.cn/dev/go/settings" +) + +type GRPCConfig struct { + ListenAddr string `yaml:"listen_addr"` +} + +func DefaultGRPCConfig() GRPCConfig { + return GRPCConfig{ + ListenAddr: ":9002", + } +} + +func (sc *GRPCConfig) Load(setting *settings.Setting) error { + if err := setting.Unmarshal("grpc_server", sc); err != nil { + return err + } + + return nil +} diff --git a/management/webserver/pkg/config/log.go b/management/webserver/pkg/config/log.go new file mode 100644 index 0000000..27cbe53 --- /dev/null +++ b/management/webserver/pkg/config/log.go @@ -0,0 +1,25 @@ +package config + +import ( + "chaitin.cn/dev/go/settings" +) + +type LogConfig struct { + Output string `yaml:"output"` + Level string `yaml:"level"` +} + +func DefaultLogConfig() LogConfig { + return LogConfig{ + Output: "stdout", + Level: "info", + } +} + +func (lc *LogConfig) Load(setting *settings.Setting) error { + if err := setting.Unmarshal("log", lc); err != nil { + return err + } + + return nil +} diff --git a/management/webserver/pkg/config/server.go b/management/webserver/pkg/config/server.go new file mode 100644 index 0000000..f9d2505 --- /dev/null +++ b/management/webserver/pkg/config/server.go @@ -0,0 +1,26 @@ +package config + +import ( + "chaitin.cn/dev/go/settings" +) + +type ServerConfig struct { + ListenAddr string `yaml:"listen_addr"` + DevMode bool `yaml:"dev_mode"` + IntenseMode bool `yaml:"intense_mode"` +} + +func DefaultServerConfig() ServerConfig { + return ServerConfig{ + ListenAddr: ":9001", + DevMode: false, + } +} + +func (sc *ServerConfig) Load(setting *settings.Setting) error { + if err := setting.Unmarshal("server", sc); err != nil { + return err + } + + return nil +} diff --git a/management/webserver/pkg/config/telemetry.go b/management/webserver/pkg/config/telemetry.go new file mode 100644 index 0000000..4960cb7 --- /dev/null +++ b/management/webserver/pkg/config/telemetry.go @@ -0,0 +1,23 @@ +package config + +import ( + "chaitin.cn/dev/go/settings" +) + +type TelemetryConfig struct { + Addr string `yaml:"addr"` +} + +func DefaultTelemetryConfig() TelemetryConfig { + return TelemetryConfig{ + Addr: "", + } +} + +func (t *TelemetryConfig) Load(setting *settings.Setting) error { + if err := setting.Unmarshal("telemetry", t); err != nil { + return err + } + + return nil +} diff --git a/management/webserver/pkg/constants/constants.go b/management/webserver/pkg/constants/constants.go new file mode 100644 index 0000000..bf729ce --- /dev/null +++ b/management/webserver/pkg/constants/constants.go @@ -0,0 +1,15 @@ +package constants + +const ( + SuperUser = "admin" + ProductName = "长亭雷池 WAF" + ProductVersion = "9.3.9" + ConfigFilePath = "config.yml" + CertsPath = "certs" +) + +const ( + NotUpgrade int = iota + RecommendedUpgrade + MustUpgrade +) diff --git a/management/webserver/pkg/constants/detectlog.go b/management/webserver/pkg/constants/detectlog.go new file mode 100644 index 0000000..962985d --- /dev/null +++ b/management/webserver/pkg/constants/detectlog.go @@ -0,0 +1,521 @@ +package constants + +const ( + ProtocolHTTP = 0 + ProtocolHTTPS = 1 + ProtocolHTTP2 = 2 +) + +var ( + HTTPProtocol = map[int]string{ + ProtocolHTTP: "http", + ProtocolHTTPS: "https", + // 在 WAF 中开启 HTTP2 的前提是开启 TLS，所以这里先把 HTTP2 都兼容显示为 HTTPS + ProtocolHTTP2: "https", + } + + AttackType = map[int]string{ + -4: "超长数据", + -3: "黑名单", + -2: "白名单", + -1: "非攻击", + 0: "SQL 注入", + 1: "XSS", + 2: "CSRF", + 3: "SSRF", + 4: "拒绝服务", + 5: "后门", + 6: "反序列化", + 7: "代码执行", + 8: "代码注入", + 9: "命令注入", + 10: "文件上传", + 11: "文件包含", + 12: "重定向", + 13: "权限不当", + 14: "信息泄露", + 15: "未授权访问", + 16: "不安全的配置", + 17: "XXE", + 18: "XPath 注入", + 19: "LDAP 注入", + 20: "目录穿越", + 21: "扫描器", + 22: "水平权限绕过", + 23: "垂直权限绕过", + 24: "文件修改", + 25: "文件读取", + 26: "文件删除", + 27: "逻辑错误", + 28: "CRLF 注入", + 29: "模板注入", + 30: "点击劫持", + 31: "缓冲区溢出", + 32: "整数溢出", + 33: "格式化字符串", + 34: "条件竞争", + 35: "HTTP 协议违规", + 61: "超时", + 62: "未知", + 63: "威胁情报", + 64: "Cookie 篡改", + } + + CountryCode = map[string]string{ + "CN": "中国", + "AD": "安道尔", + "AE": "阿联酋", + "AF": "阿富汗", + "AG": "安提瓜和巴布达", + "AI": "安圭拉", + "AL": "阿尔巴尼亚", + "AM": "亚美尼亚", + "AO": "安哥拉", + "AQ": "南极洲", + "AR": "阿根廷", + "AS": "美属萨摩亚", + "AT": "奥地利", + "AU": "澳大利亚", + "AW": "阿鲁巴", + "AX": "奥兰", + "AZ": "阿塞拜疆", + "BA": "波斯尼亚和黑塞哥维那", + "BB": "巴巴多斯", + "BD": "孟加拉国", + "BE": "比利时", + "BF": "布基纳法索", + "BG": "保加利亚", + "BH": "巴林", + "BI": "布隆迪", + "BJ": "贝宁", + "BL": "圣巴泰勒米", + "BM": "百慕大", + "BN": "文莱", + "BO": "玻利维亚", + "BQ": "加勒比荷兰", + "BR": "巴西", + "BS": "巴哈马", + "BT": "不丹", + "BV": "布韦岛", + "BW": "博茨瓦纳", + "BY": "白俄罗斯", + "BZ": "伯利兹", + "CA": "加拿大", + "CC": "科科斯（基林）群岛", + "CD": "刚果（金）", + "CF": "中非", + "CG": "刚果（布）", + "CH": "瑞士", + "CI": "科特迪瓦", + "CK": "库克群岛", + "CL": "智利", + "CM": "喀麦隆", + "CO": "哥伦比亚", + "CR": "哥斯达黎加", + "CU": "古巴", + "CV": "佛得角", + "CW": "库拉索", + "CX": "圣诞岛", + "CY": "塞浦路斯", + "CZ": "捷克", + "DE": "德国", + "DJ": "吉布提", + "DK": "丹麦", + "DM": "多米尼克", + "DO": "多米尼加", + "DZ": "阿尔及利亚", + "EC": "厄瓜多尔", + "EE": "爱沙尼亚", + "EG": "埃及", + "EH": "阿拉伯撒哈拉民主共和国", + "ER": "厄立特里亚", + "ES": "西班牙", + "ET": "埃塞俄比亚", + "FI": "芬兰", + "FJ": "斐济", + "FK": "福克兰群岛", + "FM": "密克罗尼西亚联邦", + "FO": "法罗群岛", + "FR": "法国", + "GA": "加蓬", + "GB": "英国", + "GD": "格林纳达", + "GE": "格鲁吉亚", + "GF": "法属圭亚那", + "GG": "根西", + "GH": "加纳", + "GI": "直布罗陀", + "GL": "格陵兰", + "GM": "冈比亚", + "GN": "几内亚", + "GP": "瓜德罗普", + "GQ": "赤道几内亚", + "GR": "希腊", + "GS": "南乔治亚和南桑威奇群岛", + "GT": "危地马拉", + "GU": "关岛", + "GW": "几内亚比绍", + "GY": "圭亚那", + "HM": "赫德岛和麦克唐纳群岛", + "HN": "洪都拉斯", + "HR": "克罗地亚", + "HT": "海地", + "HU": "匈牙利", + "ID": "印尼", + "IE": "爱尔兰", + "IL": "以色列", + "IM": "马恩岛", + "IN": "印度", + "IO": "英属印度洋领地", + "IQ": "伊拉克", + "IR": "伊朗", + "IS": "冰岛", + "IT": "意大利", + "JE": "泽西", + "JM": "牙买加", + "JO": "约旦", + "JP": "日本", + "KE": "肯尼亚", + "KG": "吉尔吉斯斯坦", + "KH": "柬埔寨", + "KI": "基里巴斯", + "KM": "科摩罗", + "KN": "圣基茨和尼维斯", + "KP": "朝鲜", + "KR": "韩国", + "KW": "科威特", + "KY": "开曼群岛", + "KZ": "哈萨克斯坦", + "LA": "老挝", + "LB": "黎巴嫩", + "LC": "圣卢西亚", + "LI": "列支敦士登", + "LK": "斯里兰卡", + "LR": "利比里亚", + "LS": "莱索托", + "LT": "立陶宛", + "LU": "卢森堡", + "LV": "拉脱维亚", + "LY": "利比亚", + "MA": "摩洛哥", + "MC": "摩纳哥", + "MD": "摩尔多瓦", + "ME": "黑山", + "MF": "法属圣马丁", + "MG": "马达加斯加", + "MH": "马绍尔群岛", + "MK": "马其顿", + "ML": "马里", + "MM": "缅甸", + "MN": "蒙古", + "MP": "北马里亚纳群岛", + "MQ": "马提尼克", + "MR": "毛里塔尼亚", + "MS": "蒙特塞拉特", + "MT": "马耳他", + "MU": "毛里求斯", + "MV": "马尔代夫", + "MW": "马拉维", + "MX": "墨西哥", + "MY": "马来西亚", + "MZ": "莫桑比克", + "NA": "纳米比亚", + "NC": "新喀里多尼亚", + "NE": "尼日尔", + "NF": "诺福克岛", + "NG": "尼日利亚", + "NI": "尼加拉瓜", + "NL": "荷兰", + "NO": "挪威", + "NP": "尼泊尔", + "NR": "瑙鲁", + "NU": "纽埃", + "NZ": "新西兰", + "OM": "阿曼", + "PA": "巴拿马", + "PE": "秘鲁", + "PF": "法属波利尼西亚", + "PG": "巴布亚新几内亚", + "PH": "菲律宾", + "PK": "巴基斯坦", + "PL": "波兰", + "PM": "圣皮埃尔和密克隆", + "PN": "皮特凯恩群岛", + "PR": "波多黎各", + "PS": "巴勒斯坦", + "PT": "葡萄牙", + "PW": "帕劳", + "PY": "巴拉圭", + "QA": "卡塔尔", + "RE": "留尼汪", + "RO": "罗马尼亚", + "RS": "塞尔维亚", + "RU": "俄罗斯", + "RW": "卢旺达", + "SA": "沙特阿拉伯", + "SB": "所罗门群岛", + "SC": "塞舌尔", + "SD": "苏丹", + "SE": "瑞典", + "SG": "新加坡", + "SH": "圣赫勒拿", + "SI": "斯洛文尼亚", + "SJ": "挪威", + "SK": "斯洛伐克", + "SL": "塞拉利昂", + "SM": "圣马力诺", + "SN": "塞内加尔", + "SO": "索马里", + "SR": "苏里南", + "SS": "南苏丹", + "ST": "圣多美和普林西比", + "SV": "萨尔瓦多", + "SX": "荷属圣马丁", + "SY": "叙利亚", + "SZ": "斯威士兰", + "TC": "特克斯和凯科斯群岛", + "TD": "乍得", + "TF": "法属南方和南极洲领地", + "TG": "多哥", + "TH": "泰国", + "TJ": "塔吉克斯坦", + "TK": "托克劳", + "TL": "东帝汶", + "TM": "土库曼斯坦", + "TN": "突尼斯", + "TO": "汤加", + "TR": "土耳其", + "TT": "特立尼达和多巴哥", + "TV": "图瓦卢", + "TZ": "坦桑尼亚", + "UA": "乌克兰", + "UG": "乌干达", + "UM": "美国本土外小岛屿", + "US": "美国", + "UY": "乌拉圭", + "UZ": "乌兹别克斯坦", + "VA": "梵蒂冈", + "VC": "圣文森特和格林纳丁斯", + "VE": "委内瑞拉", + "VG": "英属维尔京群岛", + "VI": "美属维尔京群岛", + "VN": "越南", + "VU": "瓦努阿图", + "WF": "瓦利斯和富图纳", + "WS": "萨摩亚", + "YE": "也门", + "YT": "马约特", + "ZA": "南非", + "ZM": "赞比亚", + "ZW": "津巴布韦", + } + + RuleModule = map[string]string{ + "m_sqli": "SQL 注入检测模块", + "m_xss": "XSS 检测模块", + "m_csrf": "CSRF 检测模块", + "m_ssrf": "SSRF 检测模块", + "m_php_unserialize": "PHP 反序列化检测模块", + "m_java_unserialize": "Java 反序列化检测模块", + "m_file_upload": "文件上传攻击检测模块", + "m_file_include": "文件包含攻击检测模块", + "m_php_code_injection": "PHP 代码注入检测模块", + "m_java": "Java 代码注入检测模块", + "m_cmd_injection": "命令注入检测模块", + "m_response": "服务器响应检测模块", + "m_scanner": "机器人检测模块", + "m_http": "畸形 HTTP 协议检测模块", + "m_asp_code_injection": "ASP 代码注入检测模块", + "m_ssti": "模板注入检测模块", + "m_rule": "通用漏洞模块", + "m_timeout": "", // 检测超时 + "whitelist": "白名单", + "blacklist": "黑名单", + } + + // RuleReason TODO: get from libfusion.so, refer to skyview `Fusion.get_rule_detail_dict()` + RuleReason = map[string]string{ + "6f4922f45568161a8cdf4ad2299f6d23": "访问测试文件的请求", + "c6b99e08e56911ecb18f00155d694977": "[HW2021] 深信服终端检测平台远程命令执行漏洞(CNVD-2020-46552)", + "c9a2f2f5b1035a1ca3a5aa776914e393": "[HW2020] GitLab 文件读写 (CVE-2017-0915, CVE-2016-9086)", + "97e264c2235c4ad0aa61ecd68fa53351": "jenkins 管理员权限开放 (CVE-2018-1999001)", + "ee41775801ab11ed90a200155ddb8f4e": "Apache HTTPD SSRF (CVE-2021-40438)", + "33e75ff09dd601bbe69f351039152189": "访问系统文件的请求", + "1c383cd30b7c298ab50293adfecb7b18": "Struts2 Java 代码注入漏洞", + "eff4046e0b0411ed976f00155dd454dd": "蓝凌OA漏洞", + "a1d0c6e83f027327d8461063f4ac58a6": "webshell", + "d92ebbf7683c5cafbc74456c73bc5d0c": "[HW2020] JIRA OAuth SSRF (CVE-2017-9506)", + "dbc22aa2fce011eca0e800163e345065": "[HW2020] Nginx range filter overflow (CVE-2017-7529)", + "ae971210f5f411ecae9200163e345065": "[HW2022] VMware 认证绕过漏洞 (CVE-2022-22972)", + "5af4619c46045568b4d701ad5208eee0": "[HW2020] Druid 未授权访问", + "fbd7939d674997cdb4692d34de8633c4": "Jackson 反序列化 (CVE-2017-17485)", + "63400573dc155253869bc90e341708e9": "[HW2020] Apache Spark 未授权访问", + "37274a94eadf11eca1a600155dcfc445": "[HW2022] Apache Solr SSRF漏洞 (CVE-2021-27905)", + "c4ca4238a0b923820dcc509a6f75849b": "PHP 代码泄露", + "3416a75f4cea9109507cacd8e2f2aefc": "phpinfo 泄露", + "c9f0f895fb98ab9159f51fd0297e236d": "访问敏感文件的请求", + "36564acfab005dcda413a0b489ce8a02": "[HW2020] ffmpeg SSRF (CVE-2016-1898)", + "31a7ad75521152eca43e7eb15065e850": "[HW2020] Kibana远程代码执行(CVE-2019-7609)", + "03d0598da679b4d68089730de888a945": "Laravel Debug 远程代码执行 (CVE-2021-3129)", + "34173cb38f07f89ddbebc2ac9128303f": "XML 实体注入漏洞", + "09afe350746411eda37500163e12b978": "Apache Kylin 未授权配置 (CVE-2020-13927)", + "4096d530f60c11eca0ad00163e345065": "[HW2020] Horde Groupware Webmail Edition 远程命令执行漏洞", + "c6b57d8ce56911ec980b00155d694977": "[HW2021] F5 BIG-IP 远程代码执行漏洞(CVE-2020-5902)", + "8964339e01b411ed9bdc00163e345065": "Spring Cloud Config 路径穿越漏洞 (CVE-2020-5405)", + "e336d501c94e556d8fa3f6df584e88e2": "[HW2020] Spring WebFlow RCE (CVE-2017-4971)", + "c6b9fa2ee56911ecb64f00155d694977": "[HW2021] 通达OA权限提升漏洞(11.5.200417 之前的版本)", + "8f1c07fe747c11ed8b7c00163e12b978": "Microsoft Exchange 远程代码执行 (CVE-2020-0688)", + "3ef815416f775098fe977004015c6193": "Apache Solr 远程代码执行漏洞 (CVE-2019-0193)", + "2e7f58dd541d5c478fc12bc8a64bef41": "[HW2020] S2-005 (2) (CVE-2010-1870)", + "1ff1de774005f8da13f42943881c655f": "DEDECMS add suffix", + "458c1b2af1f911ecafdb00163e345065": "[HW2021] Sitecore XP远程代码执行漏洞(CVE-2021-42237)", + "ac3d6ee2714b11ed87ba00163e12b978": "通达OA V11.x远程代码执行漏洞", + "9fa20cc8eabf11ec808600155dcfc445": "[HW2022] 360天擎信息泄露", + "2a1f6e4292ee30c1a92d3e8333471534": "Jackson 反序列化", + "e404c364f1f511ec9cc500163e345065": "[HW2021] Zabbix 5.0.17-Remote Code Execution (RCE) (Authenticated)", + "c6b8cd66e56911ec8cc100155d694977": "[HW2021] 宝塔面板数据库管理未授权访问漏洞(Linux正式版7.4.2、Linux测试版7.5.13、Windows正式版6.8)", + "3c27c530751511edbaf200163e12b978": "Citrix 未授权访问 ", + "22aaa784da604abeb19b0aeee2b3cc6d": "Jenkins 非预期方法调用漏洞(SECURITY-595)", + "a5bfc9e07964f8dddeb95fc584cd965d": "Java 畸形 double 数据 DOS 漏洞", + "c6babe8ce56911eca76200155d694977": "[HW2021] 致远OA 文件上传漏洞(V8.0、V7.1、V7.1SP1、V7.0、V7.0SP1、V7.0SP2、V7.0SP3、V6.0、V6.1SP1、V6.1SP2、V5.x)", + "d7e82c628f96541c924637f711555266": "[HW2020] PHPStudy 命令执行", + "7f39f8317fbdb1988ef4c628eba02591": "访问 IIS 短文件名 / 文件夹", + "44f683a84163b3523afe57c2e008bc8c": "扫描 IIS 短文件名 / 文件夹", + "c38555f8746411eda4a400163e12b978": "快排CMS 未授权访问 ", + "866cbe77d2884225a8fa19793cc8ad51": "ghostscript 命令执行 (CVE-2018-19475)", + "17400b535f9f5db5a8424265ed8cc38c": "[HW2020] JeeCMS SSRF", + "774b89ca853942cabc8b94bfff46d73c": "ECShop 2.x-3.x 远程代码执行漏洞", + "b801a4a10a22422dff885812a5d0d417": "科蓝反序列化漏洞", + "6c991e5e23215c589a86f13ae800c616": "[HW2020] Couchdb 垂直越权 (1) (CVE-2017-12635)", + "7d5a620cf69311ecbf8400163e345065": "[HW2020] ShardingShpere远程命令执行漏洞 (CVE-2020-1947)", + "a26afe82eade11ecb85f00155dcfc445": "[HW2022] Alibaba Nacos认证绕过", + "3343e2def60b11ec964400163e345065": "[HW2021] 用友ERP-NC 任意文件读取漏洞 (2021-?-?)", + "09189cf61c0e4774a4d8af7bfb0cf6de": "WebLogic wls9-async 和 wls-wsat 反序列化 (CNVD-C-2019-48814)", + "738430de6f054031a7597c00eb1fd061": "Drupal Mail 命令注入", + "2abf53869fd05cf5a69c11ea42d2cb54": "[HW2020] ActiveMQ 任意文件写入漏洞 (CVE-2016-3088)", + "e2c420d928d4bf8ce0ff2ec19b371514": "Drupal 内核远程代码执行漏洞 (CVE-2018-7602)", + "ac61a980eae411ec985100155dcfc445": "[HW2022] Zyxel NBG2105 身份验证绕过 (CVE-2021-3297)", + "d67d8ab4f4c10bf22aa353e27879133c": "PHP168 代码执行漏洞", + "e7629f3e750a11eda3a900163e12b978": "PbootCms v3.1.2 远程代码执行 ", + "6d85a117690c32af50f9b47f98abf408": "骑士 CMS 远程代码执行", + "4ebd2f6aee2811ec8e2600163e345065": "[HW2022] 齐治堡垒机任意用户登录漏洞", + "688142e95d6d446ee901de2b4d087a5d": "F5 BIG-IP漏洞", + "91d18ff6f2dc54618b1a28f15afe572f": "[HW2020] Couchdb 命令执行 (CVE-2017-12636)", + "70efdf2ec9b086079795c442636b55fb": "访问管理后台的请求", + "6d6bcd8ebc2c4d4a961ce2e8b4b3af67": "joomla 远程代码执行", + "c6b4be74e56911eca25c00155d694977": "[HW2021] Apache Shiro身份认证绕过漏洞(CVE-2020-17523)", + "1f0822d21d0a11edad0900163e345065": "[HW2020] Couchdb 垂直越权 (1) (CVE-2017-12635)", + "2024042cf11511ecb17e00163e345065": "[HW2021] GoAhead Server 环境变量注入漏洞(CVE-2021-42342)", + "abade038ebc211ec8a8800163e345065": "[HW2022] VMware SSRF XSS)", + "32ac3f49feb65bada13cdf85722d237a": "[HW2020] Spring RCE (CVE-2018-1270)", + "43ec517d68b6edd3015b3edc9a11367b": "Spring 框架漏洞", + "6a48a52c11424cd68e89d26d212556c4": "ThinkPHP5 任意代码执行漏洞", + "a5d9fe3b0172b4625a1fe6a41482aa8b": "Xstream 反序列化", + "072b030ba126b2f4b2374f342be9ed44": "路径穿越攻击", + "38529f8eebc211eca87c00163e345065": "[HW2022] VMware 任意文件读取", + "3a3128d0e7e811ec909000155db4c628": "Java 代码注入", + "e369853df766fa44e1ed0ff613f563bd": "Struts2 S2-020", + "d3d9446802a44259755d38e6d163e820": "访问 Git 仓库的请求", + "6e4b3bcf45295dcc9b1f3efef4353287": "[HW2020] JIRA SSRF (CVE-2019-8451)", + "b58a180cf11111ec8a4600163e345065": "[HW2022] VMware 服务端模板注入漏洞(CVE-2022-22954)", + "7cbbc409ec990f19c78c75bd1e06f215": "Jenkins 远程代码执行漏洞 (CVE-2018-1000861)", + "c6b6f9d2e56911ec8e2e00155d694977": "[HW2021] phpStudy nginx 解析漏洞(phptsuy8.1.07的Nginx1.5.11版本)", + "b4b143a12d0d5eeabef67b6597dc7cd9": "[HW2020] DedeCMS 密码重置漏洞", + "028c274cfc4f11ecaedd00163e345065": "[HW2022] Oracle Access Manager RCE (CVE-2021-35587)", + "1fc707c92da6519e9337f202ad1fd959": "[HW2020] 通达OA 前台任意用户登录", + "2efd7872752f11ed97f600163e12b978": "XStream 反序列化代码执行 (CVE-2021-29505)", + "9355da5d436e52cba6fd2012ad5a5832": "[HW2020] Spring Data Rest RCE (CVE-2017-8046)", + "d09bf41544a3365a46c9077ebb5e35c3": "Jackson 反序列化 (CVE-2017-7525)", + "c20ad4d76fe97759aa27a0c99bff6710": "访问 SVN 仓库的请求", + "e1c17c16169b58078be071b37eda9f3c": "[HW2020] HFS RCE", + "e4a2355fa844484bbdf799cc8795b415": "Apache Druid 远程命令执行漏洞", + "72c04a1fbd6e5641bb33d0185dc36b9b": "[HW2020] Nexus 后台表达式注入 (CVE-2020-10199, CVE-2019-7238)", + "c6b519e6e56911ecaa1c00155d694977": "[HW2021] Drupal任意PHP代码执行漏洞(CVE-2020-28948/28949)", + "c81e728d9d4c2f636f067f89cc14862c": "访问敏感文件的请求", + "a02582663945fa213788203624eb3b89": "Jackson-databind 反序列化 (CVE-2020-35790/CVE-2020-35491)", + "45c48cce2e2d7fbdea1afc51c7c6ad26": "访问 Git 仓库的请求", + "029e67ff376659daaea612ec5c3964ab": "[HW2020] Gitea LFS RCE", + "e52769085be711ec859e00155dae391c": "Apache Log4j 远程命令执行漏洞", + "19ca14e7ea6328a42e0eb13d585e4c22": "Java 代码注入漏洞 (针对 Struts 2 漏洞的通用防御)", + "3c59dc048e8850243be8079a5c74d079": "CVE-2012-1823 PHP FastCGI 远程代码执行漏洞", + "14766b63abf447ac808b92535dde9c09": "Gogs、Gitea 远程代码执行漏洞 (CVE-2018-18925)", + "9bf31c7ff062936a96d3c8bd1f8f2ff3": "后门", + "ae0196baf5f211ec98c400163e345065": "[HW2022] Sunlogin-11.0.0.33-RCE-1_获取token", + "2838023a778dfaecdc212708f721b788": "访问备份或临时文件的请求", + "110ad086eaff11ec87c500155d143f7b": "[HW2022] Gitlab exiftool 远程命令执行漏洞 (CVE-2021-22205)", + "e3959e0e30494d6caa1e9b259073d96c": "南方数据管理员添加未授权访问漏洞", + "98f13708210194c475687be6106a3b84": "Nginx code 解析漏洞", + "887d1820a85a43cfbeee578a02f6a914": "fastjson 反序列化代码执行", + "88a4a7227e81d064b551f52f7f51e6c8": "泛微 E-cology OA远程代码执行", + "4c15abcbe6e8410aac559817925bfd11": "TBK DVR 验证绕过漏洞 (CVE-2018-9995)", + "460072c82cca58999281247426c71bf8": "[HW2020] ffmpeg 文件读取 (CVE-2016-1897)", + "96c4bc6a6adf4434b93d714106f4c67f": "ghostscript 任意文件读写 (CVE-2018-17961)", + "9dbe327d8dd64f3a8d919cf19d5e3e00": "ThinkPHP 5.0.x _method 代码注入", + "6512bd43d9caa6e02c990b0a82652dca": "访问 SVN 仓库的请求", + "a12856ca51bb5ed69b6646dafd605e7c": "[HW2020] Apache Shiro 反序列化攻击 (CVE-2016-4437)", + "9473dea2eb144fee9ddeac33b49358b9": "Nexus Repository Manager 3 远程代码执行漏洞(CVE-2019-7238)", + "eccbc87e4b5ce2fe28308fd9f2a7baf3": "Apache 目录遍历", + "e799871cf6b653db94b759b79deefe5d": "[HW2020] ImageMagick RCE (CVE-2016-3714)", + "ca037bccfa7c5e7b822ea1889ba7be47": "[HW2020] MetInfo 前台 SQL 注入 (CNVD-2018-20024)", + "638d9bb801b611eda81e00163e345065": "Atlassian Jira 服务端请求伪造漏洞 (CVE-2022-26135)", + "8ef24d7501e85d1fa3ce4016dbf297d1": "[HW2020] Webmin RCE (CVE-2019-15107)", + "81f360f5033c55179d34b37b61389503": "[HW2020] Palo Alto GlobalProtect SSL RCE (CVE-2019-1579)", + "c374fb01134a4213be9ba67539523162": "IIS 扩展名漏洞 (CVE-2009-4444)", + "21203602eafd11ec8dc300155d143f7b": "[HW2022] F5 BIG-IP 远程代码执行漏洞 (CVE-2021-22986)", + "2cba1bd57c9603c07ff4f09f8e4fb0f9": "Apache AXIS 远程命令执行漏洞", + "a0ff28f2f5f111eca2de00163e345065": "[HW2022] Weblogic Server 信息泄漏漏洞利用", + "d645920e395fedad7bbbed0eca3fe2e0": "DEDECMS SQL 注入漏洞", + "2b6e15ac747811eda03000163e12b978": "Apache DolphinScheduler 远程代码执行 (CVE-2020-11974)", + "1f5449318bbc38d200fb9f3bcf87a50f": "WebLogic console 远程代码执行漏洞 (CVE-2020-14882)", + "02e74f10e0327ad868d138f2b4fdd6f0": "利用 Apache 解析漏洞来执行 PHP 脚本", + "3ff4aed51a7c506087c0b9df1b6ac0d3": "[HW2020] Couchdb 垂直越权 (2) (CVE-2017-12635)", + "933cbfbbc71c417ab9d757e5dc4be4d4": "Jenkins Accept-Language 信息泄露 (CVE-2018-1999002)", + "182be0c5cdcd5072bb1864cdee4d3d6e": "Struts2 S2-016", + "6af6b17ceae711ecbd5100155dcfc445": "[HW2022] 三星 WLAN AP WEA453e路由器 RCE (query)", + "7f41d19e081411edaca100163e345065": "[HW2022] 三星 WLAN AP WEA453e路由器 RCE (form)", + "cfcd208495d565ef66e7dff9f98764da": "JSP 代码泄露", + "3b57f3413df14b2380ae8e483cb0dcd5": "Drupalgeddon2 - Drupal核心远程代码执行", + "6364d3f0f495b6ab9dcf8d3b5c6e0b01": "Struts2 S2-005", + "a98e8c0cf12211ecb1d800163e345065": "[HW2021] VMware vCenter Server 远程代码执行漏洞(CVE-2021-21985)", + "b42530a1ea025fa78bb0c6c7d2e1ed95": "[HW2020] PHP FPM RCE (CVE-2019-11043)", + "bbe199e1efec469588fde16dce75df36": "coremail 信息泄露", + "c6b7547ce56911ec9d3d00155d694977": "[HW2021] SaltStack多个高危漏洞(CVE-2020-16846,CVE-2020-17490,CVE-2020-25592)", + "c6b7ae5ee56911ec9d1600155d694977": "[HW2021] SAP NetWeaver AS JAVA 高危漏洞(CVE-2020-6287)", + "65d86160f1f711ec971800163e345065": "[HW2021] Total.js框架远程代码执行漏洞(CVE-2021-23389)", + "aa4755eb0fdf47ec91e8f285613e7954": "Atlassian 漏洞", + "c6b5dc64e56911ecbe7300155d694977": "[HW2021] Microsoft Exchange远程代码执行漏洞(CVE-2020-16875)", + "b6f76f653ded54b494292eb6c285d464": "[HW2020] WebLogic 弱口令", + "9778d5d219c5080b9a6a17bef029331c": "Apache Solr 远程代码执行漏洞 (CVE-2019-0192)", + "c6bb1f3ae56911ec93a500155d694977": "[HW2021] WebSphere 权限提升漏洞(CVE-2020-4276)", + "c6ba5e10e56911ec8f6d00155d694977": "[HW2021] 微软 SQL Server 报表服务远程代码执行漏洞(CVE-2020-0618)", + "061de838eabf11ec8ee500155dcfc445": "[HW2022] 360天擎SQL注入漏洞", + "6ea9ab1baa0efb9e19094440c317e21b": "XML 实体注入漏洞", + "039c966635bd4ac286553f55d9457615": "dedecms 5.7 升级时间泄露", + "b6d767d2f8ed5d21a44b0e5886680cb9": "DEDECMS add suffix", + "f033ab37c30201f73f142449d037028d": "ActiveMQ 任意文件写入漏洞 (CVE-2016-3088)", + "558fce096ca65beb8f7ca7d4b2bf6b4f": "[HW2020] ElasticSearch 命令执行 (CVE-2015-1427)", + "b53b3a3d6ab90ce0268229151c9bde11": "代码注入漏洞 (针对 WordPress)", + "c6b4239ce56911ec8b8a00155d694977": "[HW2021] Apache Shiro 权限绕过漏洞(CVE-2020-13933)", + "2408e35abe40580286b6d9106b43357f": "[HW2020] S2-005 (1) (CVE-2010-1870)", + "724390fd0bf29ab3adb92ee026474fe1": "Jackson-databind 反序列化 (CVE-2020-36179~CVE-2020-36189)", + "ef268f2b31fd5943bd1882e2218e9924": "[HW2020] Tomcat RCE (CVE-2019-0232)", + "d49bb7ebe41146deb48138f28d220c7f": "可疑远程调用协议", + "aab3238922bcc25a6f606eb525ffdc56": "phpinfo 泄露", + "582e036cee2811ec90c700163e345065": "[HW2022] H3C SecPath运维审计系统任意用户登录漏洞", + "25d513abf19740b083f2262270c0a281": "PHPCMS v9.6.0 任意文件上传", + "ab299838fbe1584d843fc004b0034e81": "[HW2020] GoAhead RCE (CVE-2017-17562)", + "c6b81b78e56911ec989800155d694977": "[HW2021] SolarWinds 远程代码执行漏洞(CVE-2020-10148)", + "daf7d4a0116711ed8d9c00163e35b03e": "Seacms php代码注入", + "c6b93e5ee56911ec8e1500155d694977": "[HW2021] 深信服终端检测平台任意用户登录(CNVD-2020-46552)", + "95a4035353ed405ba3418698fc06b544": "spring 目录遍历 (cve-2014-3625)", + "eab53f221ee611edb57e00163e345065": "VMware Workspace ONE UEM SSRF (CVE-2021-22054)", + "fca34064ebc211ec9b9900163e345065": "[HW2022] VMware vRealize SSRF (CVE-2021-21975)", + "c51ce410c124a10e0db5e4b97fc2af39": "PHPSpy 后门", + "2f7e97b9204b5239b089c5ab83242519": "[HW2020] ElasticSearch 未授权访问", + "6e34a9cb6f5b43e4973b170e626a2f77": "Apache APISIX lua 远程代码执行", + "299ce1c43af85c7987d8ddef915888a8": "[HW2020] Zimbra RCE", + "529d97a8f5d711eca1e700163e345065": "[HW2022] Weblogic Server 信息泄漏漏洞利用 (CVE-2022-21371)", + "49234868ee1f11ec833b00155de98ae1": "[HW2022] AVTECH监控后台命令注入漏洞", + "1f0e3dad99908345f7439f8ffabdffc4": "PHP FastCGI 解析漏洞", + } +) diff --git a/management/webserver/pkg/constants/detector.go b/management/webserver/pkg/constants/detector.go new file mode 100644 index 0000000..11b68ed --- /dev/null +++ b/management/webserver/pkg/constants/detector.go @@ -0,0 +1,8 @@ +package constants + +const ( + ContentType = "application/octet-stream" + UpdateEntrypoint = "/update/policy" + StatEntrypoint = "/stat" + DefaultPolicyVersion = "1" +) diff --git a/management/webserver/pkg/constants/option.go b/management/webserver/pkg/constants/option.go new file mode 100644 index 0000000..e959897 --- /dev/null +++ b/management/webserver/pkg/constants/option.go @@ -0,0 +1,8 @@ +package constants + +const ( + SecretKey = "secret_key" + MachineID = "machine_id" + PolicyGroupGlobal = "policy_group_global" + SrcIPConfig = "src_ip_config" +) diff --git a/management/webserver/pkg/constants/session.go b/management/webserver/pkg/constants/session.go new file mode 100644 index 0000000..a789aa5 --- /dev/null +++ b/management/webserver/pkg/constants/session.go @@ -0,0 +1,5 @@ +package constants + +const ( + DefaultSessionUserKey = "user" +) diff --git a/management/webserver/pkg/constants/telemetry.go b/management/webserver/pkg/constants/telemetry.go new file mode 100644 index 0000000..e8684a3 --- /dev/null +++ b/management/webserver/pkg/constants/telemetry.go @@ -0,0 +1,12 @@ +package constants + +const ( + ApplicationJson = "application/json" + TelemetryEntryPoint = "/telemetry" + TelemetryId = "9e88109c-ebbf-4d82-8a25-f97f2ac5f3c4" + Behaviour = "behaviour" + FalsePositives = "false_positives" + Installation = "Installation" +) + +var Version string diff --git a/management/webserver/pkg/cron/cron.go b/management/webserver/pkg/cron/cron.go new file mode 100644 index 0000000..46abdd4 --- /dev/null +++ b/management/webserver/pkg/cron/cron.go @@ -0,0 +1,26 @@ +package cron + +import ( + "github.com/robfig/cron/v3" + + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/log" +) + +var logger = log.GetLogger("cron") + +func newCronWithSeconds() *cron.Cron { + secondParser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | + cron.Dom | cron.Month | cron.DowOptional | cron.Descriptor) + return cron.New(cron.WithParser(secondParser), cron.WithChain()) +} + +func StartCron() error { + cronInstance := newCronWithSeconds() + _, err := cronInstance.AddFunc(SpecUpdatePolicy, CheckAndUpdatePolicy) + if err != nil { + return err + } + + cronInstance.Start() + return nil +} diff --git a/management/webserver/pkg/cron/update_policy.go b/management/webserver/pkg/cron/update_policy.go new file mode 100644 index 0000000..ab5b6f8 --- /dev/null +++ b/management/webserver/pkg/cron/update_policy.go @@ -0,0 +1,114 @@ +package cron + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + "time" + + "chaitin.cn/patronus/safeline-2/management/webserver/utils" + + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" +) + +// SpecUpdatePolicy http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/tutorial-lesson-06.html +// Seconds Minutes Hours Day-of-Month Month Day-of-Week Year (optional field) +// every 5 second (starting from 0s) +const SpecUpdatePolicy = "0/5 * * * * ?" + +type statResponseBody struct { + PolicyVersion string `json:"policy_version"` +} + +func getBytecode() ([]byte, error) { + buff, err := ioutil.ReadFile(config.GlobalConfig.Detector.FslBytecode) + if err != nil { + return nil, err + } + return buff, nil +} + +func CheckAndUpdatePolicy() { + if config.GlobalConfig.Detector.Addr == "" { + return + } + + exist, err := utils.FileExist(config.GlobalConfig.Detector.FslBytecode) + if !exist || err != nil { + return + } + + data, err := getBytecode() + if err != nil { + logger.Error(err) + return + } + + reader := bytes.NewReader(data) + + tr := &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 30 * time.Second, + } + client := &http.Client{Transport: tr} + + addr := config.GlobalConfig.Detector.Addr + statReq, err := http.NewRequest(http.MethodGet, addr+constants.StatEntrypoint, nil) + if err != nil { + logger.Error(err) + return + } + + updateReq, err := http.NewRequest(http.MethodPost, addr+constants.UpdateEntrypoint, reader) + if err != nil { + logger.Error(err) + return + } + + updateReq.Header.Set("Content-Type", constants.ContentType) + + statRspData := statResponseBody{} + statRsp, err := client.Do(statReq) + if err != nil || statRsp.StatusCode != http.StatusOK { + logger.Warn(err) + return + } + + body, err := ioutil.ReadAll(statRsp.Body) + if err != nil { + logger.Warn(err) + return + } + + err = json.Unmarshal(body, &statRspData) + if err != nil { + logger.Warn(err) + return + } + + if statRspData.PolicyVersion == constants.DefaultPolicyVersion { + return + } + + err = statRsp.Body.Close() + if err != nil { + logger.Warn(err) + } + + logger.Info("Update fsl bytecode") + updateRsp, err := client.Do(updateReq) + if err != nil { + logger.Warn(err) + return + } + + if updateRsp.StatusCode != http.StatusOK { + logger.Warnf("%s update policy, return %d", addr, updateRsp.StatusCode) + } + err = updateRsp.Body.Close() + if err != nil { + logger.Warn(err) + } +} diff --git a/management/webserver/pkg/database/postgres.go b/management/webserver/pkg/database/postgres.go new file mode 100644 index 0000000..ae7aa7a --- /dev/null +++ b/management/webserver/pkg/database/postgres.go @@ -0,0 +1,63 @@ +package database + +import ( + "gorm.io/driver/postgres" + "gorm.io/gorm" + "gorm.io/gorm/logger" + "gorm.io/gorm/schema" + + "chaitin.cn/dev/go/errors" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" +) + +type PostgresDB struct { + *gorm.DB +} + +var ( + // db is not designed to be used directly, use method `GetDB` instead. + db PostgresDB +) + +func GetDB() *PostgresDB { + return &db +} + +func InitDB() error { + dbConfig := config.GlobalConfig.DB + + if dbConfig.URL == "" { + return errors.New("empty database url") + } + // URL also works, see https://godoc.org/github.com/lib/pq + var ( + gormDB *gorm.DB + err error + ) + + gormConfig := &gorm.Config{ + NamingStrategy: schema.NamingStrategy{ + TablePrefix: "mgt_", // table name prefix, table for `User` would be `t_users` + SingularTable: true, // use singular table name, table for `User` would be `user` with this option enabled + }, + AllowGlobalUpdate: false, + Logger: logger.Default.LogMode(logger.Silent), + } + if dbConfig.LogSQL { + gormConfig.Logger = logger.Default.LogMode(logger.Info) + } + + gormDB, err = gorm.Open(postgres.Open(dbConfig.URL), gormConfig) + + if err != nil { + return err + } + + db.DB = gormDB + + return nil +} + +func (db *PostgresDB) SetDB(gormDB *gorm.DB) { + db.DB = gormDB +} diff --git a/management/webserver/pkg/fvm/fsl/action.go b/management/webserver/pkg/fvm/fsl/action.go new file mode 100644 index 0000000..f33295c --- /dev/null +++ b/management/webserver/pkg/fvm/fsl/action.go @@ -0,0 +1,22 @@ +package fsl + +import ( + "fmt" + "strings" +) + +func Goto(m string) string { + return fmt.Sprintf("GOTO %s", m) +} + +func Wheres(items ...string) string { + return strings.Join(items, " AND ") +} + +func Actions(items ...string) string { + return strings.Join(items, ", ") +} + +func Set(k, t, v string) string { + return fmt.Sprintf("SET %s::%s = %s", k, t, v) +} diff --git a/management/webserver/pkg/fvm/fsl/quote.go b/management/webserver/pkg/fvm/fsl/quote.go new file mode 100644 index 0000000..fdfea49 --- /dev/null +++ b/management/webserver/pkg/fvm/fsl/quote.go @@ -0,0 +1,50 @@ +package fsl + +import ( + "fmt" +) + +// see : https://chaitin.cn/patronus/fvm/-/blob/master/src/util/StrUtil.cpp#L18 + +func appendEscapedByte(buf []byte, b byte) []byte { + if b >= 0x20 && b <= 0x7e && b != '\'' && b != '"' && b != '\\' { + return append(buf, b) + } + switch b { + case '\'': + return append(buf, `\'`...) + case '"': + return append(buf, `\"`...) + case '\n': + return append(buf, `\n`...) + case '\t': + return append(buf, `\t`...) + case '\r': + return append(buf, `\r`...) + case '\b': + return append(buf, `\b`...) + case '\f': + return append(buf, `\f`...) + case '\\': + return append(buf, `\\`...) + default: + return append(buf, fmt.Sprintf("\\x%x", b)...) + } +} + +func appendQuotedWith(buf []byte, s string, quote byte) []byte { + buf = append(buf, quote) + for i := 0; i < len(s); i++ { + buf = appendEscapedByte(buf, s[i]) + } + buf = append(buf, quote) + return buf +} + +func quoteWith(s string, quote byte) string { + return string(appendQuotedWith(make([]byte, 0, 3*len(s)/2), s, quote)) +} + +func Quote(s string) string { + return quoteWith(s, '\'') +} diff --git a/management/webserver/pkg/fvm/fsl/selector.go b/management/webserver/pkg/fvm/fsl/selector.go new file mode 100644 index 0000000..c9d0f62 --- /dev/null +++ b/management/webserver/pkg/fvm/fsl/selector.go @@ -0,0 +1,38 @@ +package fsl + +import ( + "fmt" +) + +type Selector struct { + TableName string + ID string + Where string + Action string +} + +func NewSelector(tableName, id, where, action string) *Selector { + return &Selector{ + TableName: tableName, + ID: id, + Where: where, + Action: action, + } +} + +func (s *Selector) String() string { + ret := fmt.Sprintf("ID %s", Quote(s.ID)) + if s.Where != "" { + ret += " WHERE " + s.Where + } + ret += fmt.Sprintf(" ACTION %s", s.Action) + return ret +} + +func (s *Selector) AppendInto() string { + return fmt.Sprintf("APPEND INTO %s %s;", s.TableName, s.String()) +} + +func AppendInto(tableName, id, where, action string) string { + return NewSelector(tableName, id, where, action).AppendInto() +} diff --git a/management/webserver/pkg/fvm/fsl/state.go b/management/webserver/pkg/fvm/fsl/state.go new file mode 100644 index 0000000..f6f3258 --- /dev/null +++ b/management/webserver/pkg/fvm/fsl/state.go @@ -0,0 +1,26 @@ +package fsl + +import ( + "chaitin.cn/dev/go/errors" +) + +type State int + +const ( + S_NOP State = iota + S_ABORT + S_RETURN +) + +func (s State) String() string { + switch s { + case S_NOP: + return "NOP" + case S_ABORT: + return "ABORT" + case S_RETURN: + return "RETURN" + default: + panic(errors.New("wrong State")) + } +} diff --git a/management/webserver/pkg/fvm/fsl/table.go b/management/webserver/pkg/fvm/fsl/table.go new file mode 100644 index 0000000..6291f2a --- /dev/null +++ b/management/webserver/pkg/fvm/fsl/table.go @@ -0,0 +1,38 @@ +package fsl + +import ( + "fmt" + "strings" +) + +type Table struct { + Name string + Mappings map[State]State +} + +func NewTable(name string, mappings map[State]State) *Table { + return &Table{ + Name: name, + Mappings: mappings, + } +} + +func (t *Table) String() string { + ret := fmt.Sprintf("TABLE %s", t.Name) + if len(t.Mappings) > 0 { + var mappings []string + for from, to := range t.Mappings { + mappings = append(mappings, fmt.Sprintf("%s TO %s", from.String(), to.String())) + } + ret += fmt.Sprintf(" MAPPING %s", strings.Join(mappings, ", ")) + } + return ret +} + +func (t *Table) Create() string { + return fmt.Sprintf("CREATE %s;", t.String()) +} + +func CreateTable(name string, mappings map[State]State) string { + return NewTable(name, mappings).Create() +} diff --git a/management/webserver/pkg/fvm/fsl/target.go b/management/webserver/pkg/fvm/fsl/target.go new file mode 100644 index 0000000..bb0b83b --- /dev/null +++ b/management/webserver/pkg/fvm/fsl/target.go @@ -0,0 +1,35 @@ +package fsl + +import ( + "fmt" +) + +type Target struct { + ID string + Type string + Args string +} + +func NewTarget(id string, typ string, args string) *Target { + return &Target{ + ID: id, + Type: typ, + Args: args, + } +} + +func NewSkynetTarget(id, configJSONStr string) *Target { + return NewTarget(id, "skynet", configJSONStr) +} + +func (t *Target) String() string { + return fmt.Sprintf("%s TYPE %s ARGS (%s)", t.ID, t.Type, t.Args) +} + +func (t *Target) Create() string { + return fmt.Sprintf("CREATE TARGET %s;", t.String()) +} + +func CreateTarget(id, configJSONStr string) string { + return NewSkynetTarget(id, configJSONStr).Create() +} diff --git a/management/webserver/pkg/fvm/fvm.go b/management/webserver/pkg/fvm/fvm.go new file mode 100644 index 0000000..32a3439 --- /dev/null +++ b/management/webserver/pkg/fvm/fvm.go @@ -0,0 +1,267 @@ +package fvm + +/* +#cgo CFLAGS: -I../../submodule/fvm/include -I../../submodule/libct/include +#cgo LDFLAGS: -L../../submodule/fvm/lib -lfvm +#include +#include +#include +#include +#include +#include + +static inline fvm_update_t *build_update(const void* buf, size_t len) { + ct_string_t tmp = CT_STRING_FROM_PTR_LENGTH(buf, len); + return fvm_update_create(tmp); +} +*/ +import ( + "C" //nolint:typecheck +) +import ( + "crypto/tls" + "fmt" + "io" + "net/http" + "net/url" + "unsafe" + + "chaitin.cn/dev/go/errors" + "chaitin.cn/dev/go/log" +) + +// Constant for Output +const ( + OutputItemTable int = 0 + OutputItemSelector int = 1 + OutputItemTarget int = 2 +) + +// FVM wrapper +type FVM struct { + framework *C.fvm_framework_t + apiTable *C.char + version int64 +} + +type FVMUpdate struct { + update *C.fvm_update_t +} + +type FVMRe struct { + data []byte +} + +func maybePointer(array []byte) unsafe.Pointer { + if len(array) == 0 { + return nil + } + return unsafe.Pointer(&array[0]) +} + +func (u *FVMUpdate) ToBytes() []byte { + return C.GoBytes(unsafe.Pointer(u.update.buf.ptr), C.int(u.update.buf.length)) +} + +func (u *FVMUpdate) FromBytes(out []byte) error { + ptr := C.build_update(unsafe.Pointer(&out[0]), C.ulong(len(out))) + if ptr == nil { + return errors.New("failed to create update from db") + } + u.update = ptr + return nil +} + +func (u *FVMUpdate) MergeUpdate(patch *FVMUpdate) error { + m := C.fvm_update_merge(u.update, patch.update) + if m == nil { + return errors.New("failed to merge two updates") + } + C.fvm_update_destroy(u.update) + u.update = m + return nil +} + +func (u *FVMUpdate) Release() { + C.fvm_update_destroy(u.update) +} + +func (r *FVMRe) ToBytes() []byte { + return r.data +} + +func (r *FVMRe) FromBytes(o []byte) { + r.data = o +} + +type FVMOutput struct { + output *C.fvm_output_t +} + +// New creates FVM +func New() (*FVM, error) { + apiTable := C.fvm_api_table() + + fw := C.fvm_framework_acquire(nil) + if fw == nil { + return nil, errors.New("fvm_framework_acquire returns a null pointer") + } + + return &FVM{framework: fw, apiTable: apiTable, version: 0}, nil +} + +func (f *FVM) Update(upd *FVMUpdate) error { + if upd == nil || upd.update == nil { + return errors.New("null pointer passed to update") + } + ret := bool(C.fvm_update(f.framework, upd.update)) + if !ret { + return errors.New("update framework failed") + } + return nil +} + +func (f *FVM) Compile(text string, re []byte) (*FVMOutput, *FVMRe, error) { + cplPtr := C.fvm_compiler_acquire(f.apiTable) + cText := C.CString(text) + + defer C.fvm_compiler_release(cplPtr) + defer C.free(unsafe.Pointer(cText)) + + ok := C.fvm_compiler_load_re(cplPtr, (*C.char)(maybePointer(re)), C.ulong(len(re))) + if !ok { + return nil, nil, errors.New("failed to load re") + } + + outputPtr := C.fvm_compile(cplPtr, cText) + if outputPtr == nil { + return nil, nil, errors.New("compiling FSL failed") + } + + var cRePtr *C.char + var cReLen C.ulong + ok = C.fvm_compiler_dump_re(cplPtr, &cRePtr, &cReLen) + if !ok { + return nil, nil, errors.New("failed to dump re") + } + defer C.free(unsafe.Pointer(cRePtr)) + + cRe := &FVMRe{C.GoBytes(unsafe.Pointer(cRePtr), C.int(cReLen))} + + return &FVMOutput{outputPtr}, cRe, nil +} + +func (f *FVM) Plot() string { + plotP := C.fvm_plot(f.framework) + defer C.fvm_plot_destroy(plotP) + goBuf := C.GoBytes(unsafe.Pointer(plotP.buf.ptr), C.int(plotP.buf.length)) + return string(goBuf) +} + +func (f *FVM) Dump() *FVMUpdate { + return &FVMUpdate{C.fvm_dump(f.framework)} +} + +func (f *FVM) Release() { + C.fvm_framework_release(f.framework) +} + +func Serialize(out *FVMOutput, diff bool, base int64, target int64) *FVMUpdate { + updatePtr := C.fvm_serialize(out.output, C.bool(diff), C.long(base), C.long(target)) + if updatePtr == nil { + return nil + } + ctStrPtr := updatePtr.buf.ptr + if ctStrPtr == nil { + return nil + } + return &FVMUpdate{updatePtr} +} + +func ReleaseOutput(out *FVMOutput) { + C.fvm_output_destroy(out.output) +} + +func (f *FVM) PushFsl(server string, update *FVMUpdate) error { + log.Infof("Push FSL to %s", server) + + var length = uint32(update.update.buf.length) + var p = uintptr(unsafe.Pointer(update.update.buf.ptr)) + + log.Infof("Get update length %d, %d", length, C.int(update.update.buf.length)) + + u := &upload{ + Length: length, + P: p, + } + + base, _ := url.Parse(server) + base.Path = "/update/policy" + snURL := base.String() + req, err := http.NewRequest( + "POST", + snURL, + u, + ) + if err != nil { + return errors.Annotatef(err, "create request failed") + } + req.Header.Add("Content-Type", "application/octet-stream") + // req.Header.Set("Connection", "close") + + httpClient := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + + var ( + respErr error + resp *http.Response + ) + for i := 0; i < 3; i++ { + resp, respErr = httpClient.Do(req) + if respErr == nil { + break + } + } + if respErr != nil { + return errors.Annotatef(respErr, "Get response failed") + } + if resp != nil && resp.StatusCode != 200 { + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + log.Fatal(err) + } + bodyString := string(bodyBytes) + + return errors.New(fmt.Sprintf("Push FSL response(%d %s)", resp.StatusCode, bodyString)) + } + + return nil +} + +type upload struct { + Length uint32 + off uint32 + P uintptr +} + +func (u *upload) Read(p []byte) (n int, err error) { + if u.off >= u.Length { + if len(p) == 0 { + return 0, nil + } + return 0, io.EOF + } + plen := uint32(len(p)) + var i uint32 + for i = 0; i < plen; i++ { + if i+u.off >= u.Length { + break + } + p[i] = *(*byte)(unsafe.Pointer(u.P + unsafe.Sizeof(p[0])*uintptr(i+u.off))) + } + u.off = i + u.off + return int(i), nil +} diff --git a/management/webserver/pkg/fvm/generator.go b/management/webserver/pkg/fvm/generator.go new file mode 100644 index 0000000..e638333 --- /dev/null +++ b/management/webserver/pkg/fvm/generator.go @@ -0,0 +1,318 @@ +package fvm + +import ( + "encoding/json" + "fmt" + "strings" + + "gorm.io/gorm" + + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" + + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" + + "chaitin.cn/dev/go/errors" + "chaitin.cn/dev/go/log" + "chaitin.cn/patronus/safeline-2/management/webserver/model" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/fvm/fsl" +) + +const ( + Main = "main" + Preprocess = "preprocess" + MassPackage = "mass_package" + PolicyRule = "policy_rule" + PolicyRuleGlobal = "policy_rule_global" + PolicyGroup = "policy_group" + PolicyGroupDetect = "policy_group_detect" + WeblogGenerate = "weblog_generate" + ReqWeblogGenerate = "req_weblog_generate" + RspWeblogGenerate = "rsp_weblog_generate" + CommonLog = "common_log" + + STR = "STRING" + INT = "INTEGER" + INET = "INET" + BOOL = "BOOLEAN" +) + +var logger = log.GetLogger("fvm") + +func InitFVMBytecode() { + go func() { + fullFSL, err := GenerateFullFSL(database.GetDB().DB) + if err != nil { + logger.Fatalln("Failed to generate fsl", err) + } + if err = CompileAndSave(fullFSL); err != nil { + logger.Fatalln("Failed to compile and save fsl", err) + } + }() +} + +func PushFSL(tx *gorm.DB) error { + fullFSL, err := GenerateFullFSL(tx) + if err != nil { + logger.Debugf("Full fsl: %s", fullFSL) + logger.Error(err) + return err + } + if err = CompileAndPush(fullFSL, config.GlobalConfig.Detector.Addr); err != nil { + logger.Error(err) + return err + } + return nil +} + +func GenerateFullFSL(tx *gorm.DB) (fullFSL string, err error) { + logger.Info("Generate FSL") + + pFSL, err := PreprocessTable(tx) + if err != nil { + return "", err + } + fullFSL += pFSL + + fullFSL += MassPackageTable() + + prFSL, err := PolicyRuleTable(tx) + if err != nil { + return "", err + } + fullFSL += prFSL + + pgFSL, err := PolicyGroupTable(tx) + if err != nil { + return "", err + } + fullFSL += pgFSL + + fullFSL += WeblogGenerateTable() + + fullFSL += MainTable() + + return fullFSL, err +} + +func PreprocessTable(db *gorm.DB) (pFSL string, err error) { + pFSL += fsl.CreateTable(Preprocess, nil) + pFSL += fsl.AppendInto(Preprocess, "http_parse", "", "http_parse()") + pFSL += fsl.AppendInto(Preprocess, "init_variables", "", fsl.Actions( + fsl.Set("@site_uuid", STR, "get_http_all_headers('SL-CE-SUID')"), + fsl.Set("@src_ip", INET, "socket.src_ip"), + fsl.Set("@skip_remaining", BOOL, "false"), + fsl.Set("@rule_id", STR, fsl.Quote("")), + fsl.Set("@attack_type", INT, "-1"), + fsl.Set("@final_action", INT, "0"), + fsl.Set("@risk_level", INT, "0"), + fsl.Set("@host", STR, "http.decayed_host"), + fsl.Set("@start_time", INT, "now.timestamp_us"), + fsl.Set("@transport_scheme", STR, "transport.scheme"), + fsl.Set("@forward_log", INT, "0"), + fsl.Set("@convert_to_alog", INT, "1"), + fsl.Set("@alog_save_to_db", INT, "0"), + fsl.Set("@convert_to_blog", INT, "0"), + fsl.Set("@blog_save_to_db", INT, "0"), + fsl.Set("@convert_to_dlog", INT, "0"), + fsl.Set("@dlog_save_to_db", INT, "0"), + )) + pFSL += fsl.AppendInto(Preprocess, "set_host", "@host::STRING = ''", fsl.Actions(fsl.Set("@host", STR, "inet_to_string(socket.dst_ip)"))) + pFSL += fsl.AppendInto(Preprocess, "adapt_http2", "@transport_scheme::STRING = 'http2'", fsl.Actions(fsl.Set("@transport_scheme", STR, fsl.Quote("https")))) + + // get src_ip + sc, err := model.GetSrcIPConfig(db) + if err != nil { + return "", err + } + if sc.Source == model.HTTPHeader { + key := fmt.Sprintf("string_to_inet(get_http_all_headers(%s))", fsl.Quote(sc.Value)) + pFSL += fsl.AppendInto(Preprocess, "set_src_ip", fmt.Sprintf("%s != '::/0'::INET", key), fsl.Actions(fsl.Set("@src_ip", INET, key))) + } + return pFSL, nil +} + +func MassPackageTable() (mpFSL string) { + mpFSL += fsl.CreateTable(MassPackage, nil) + mpFSL += fsl.AppendInto(MassPackage, "mass_package_detect", "string.length(http.raw_req_body) > 1048576", fsl.Actions( + fsl.Set("@skip_remaining", BOOL, "true"), + fsl.Set("@rule_id", STR, fsl.Quote("mass_package")), + fsl.Set("@final_action", INT, "0"), + fsl.Set("@convert_to_dlog", INT, "1"), + fsl.Set("@dlog_save_to_db", INT, "1"), + fsl.Set("@attack_type", INT, "-4"), + )) + return mpFSL +} + +func PolicyRuleTable(db *gorm.DB) (prFSL string, err error) { + prFSL += fsl.CreateTable(PolicyRuleGlobal, map[fsl.State]fsl.State{fsl.S_ABORT: fsl.S_RETURN}) + + var policyRuleList []model.PolicyRule + res := db.Model(&model.PolicyRule{}).Where(&model.PolicyRule{IsEnabled: true}).Order("id desc").Find(&policyRuleList) + if res.Error != nil { + err = errors.New(fmt.Sprintf("Error occurred when fetch PolicyRule: %s", res.Error)) + } + + for _, policyRule := range policyRuleList { + + var patternBytes []byte + patternBytes, err = policyRule.Pattern.MarshalJSON() + if err != nil { + return "", err + } + var patterns []model.PolicyRulePattern + err = json.Unmarshal(patternBytes, &patterns) + if err != nil { + return "", err + } + + var wheres []string + for _, pattern := range patterns { + // transform key to string + var key string + switch pattern.K { + case model.KeySrcIP: + key = "inet_to_string(@src_ip::INET)" + case model.KeyURI: + key = "@uri_decoded::STRING" + case model.KeyHost: + key = "http.host" + default: + return "", errors.New("wrong Key") + } + + switch pattern.Op { + case model.OpEq: + wheres = append(wheres, fmt.Sprintf("string.equals_case(%s, %s)", key, fsl.Quote(pattern.V))) + case model.OpMatch: + // . -> \. + // * -> .* + valueQuote := strings.ReplaceAll(fsl.Quote(strings.ReplaceAll(pattern.V, ".", "\\.")), "*", ".*") + wheres = append(wheres, fmt.Sprintf("pcre_match(%s, %s)", valueQuote, key)) + case model.OpCIDR: + // use INET for CIDR compare + key = "@src_ip::INET" + wheres = append(wheres, fmt.Sprintf("%s IN CIDR(%s)", key, fsl.Quote(pattern.V))) + case model.OpHas: + wheres = append(wheres, fmt.Sprintf("string.contains_case(%s, %s)", key, fsl.Quote(pattern.V))) + case model.OpPrefix: + l := len(pattern.V) + wheres = append(wheres, fmt.Sprintf("string.equals_case(string.substr(%s, 0, %d), %s)", key, l, fsl.Quote(pattern.V))) + case model.OpRe: + wheres = append(wheres, fmt.Sprintf("pcre_match(%s, %s)", fsl.Quote(pattern.V), key)) + default: + return "", errors.New("wrong Operator") + } + } + + // save log only when deny and dry_run + logOption := "1" + attackType := "-3" + // DROP or ACCEPT + appendix := "DROP" + if policyRule.Action == 0 { + logOption = "0" + attackType = "-2" + appendix = "set_ctx_allow(), ACCEPT" // this selector seems no use + } + prFSL += fsl.AppendInto(PolicyRuleGlobal, fmt.Sprintf("%s_%d", PolicyRuleGlobal, policyRule.ID), fsl.Wheres(wheres...), fsl.Actions( + fsl.Set("@skip_remaining", BOOL, "true"), + fsl.Set("@rule_id", STR, fsl.Quote(fmt.Sprintf("/%s", policyRule.Comment))), + fsl.Set("@final_action", INT, fmt.Sprintf("%d", policyRule.Action)), + fsl.Set("@convert_to_dlog", INT, logOption), + fsl.Set("@dlog_save_to_db", INT, logOption), + fsl.Set("@attack_type", INT, attackType), + appendix, + // no use in v1.1 + //fsl.Set("@risk_level", INT, ""), + )) + } + + prFSL += fsl.CreateTable(PolicyRule, nil) + prFSL += fsl.AppendInto(PolicyRule, "preprocess", "", fsl.Actions(fsl.Set("@uri_decoded", STR, "url_decode(http.uri)"))) + prFSL += fsl.AppendInto(PolicyRule, "policy_rule_global", "", fsl.Goto(PolicyRuleGlobal)) + prFSL += fsl.AppendInto(PolicyRule, "set_status_code", "@final_action::INTEGER = 1", "set_status_code(403)") + + return prFSL, err +} + +func PolicyGroupTable(db *gorm.DB) (pgFSL string, err error) { + skynetConfigStr, err := model.GetSkynetConfig(db) + if err != nil { + return "", err + } + + pgFSL += fsl.CreateTarget("skynet_config", fsl.Quote(skynetConfigStr)) + + pgFSL += fsl.CreateTable(PolicyGroupDetect, map[fsl.State]fsl.State{fsl.S_ABORT: fsl.S_RETURN}) + pgFSL += fsl.AppendInto(PolicyGroupDetect, "do_detect", "", "target 'skynet_config' type skynet do detect()") + + pgFSL += fsl.CreateTable(PolicyGroup, nil) + pgFSL += fsl.AppendInto(PolicyGroup, "do_detect", "", fsl.Goto(PolicyGroupDetect)) + pgFSL += fsl.AppendInto(PolicyGroup, "dlog_persistence", "skynet.enable_log = true", fsl.Actions( + fsl.Set("@convert_to_dlog", INT, "1"), + fsl.Set("@dlog_save_to_db", INT, "1"), + )) + pgFSL += fsl.AppendInto(PolicyGroup, "set_status_code", "skynet.action = 1", "set_status_code(403)") + pgFSL += fsl.AppendInto(PolicyGroup, "hit_policy_group", "skynet.rule_id != ''", fsl.Actions( + fsl.Set("@skip_remaining", BOOL, "true"), + fsl.Set("@rule_id", STR, "skynet.rule_id"), + fsl.Set("@attack_type", INT, "skynet.main_attack"), + fsl.Set("@final_action", INT, "skynet.action"), + fsl.Set("@risk_level", INT, "skynet.risk_level"), + fsl.Set("@convert_to_dlog", INT, "1"), + )) + return pgFSL, nil +} + +func CommonLogTable() (clFSL string) { + clFSL += fsl.CreateTable(CommonLog, nil) + clFSL += fsl.AppendInto(CommonLog, "batch_set_payload_string", "", "weblog_payload_set_string_batch({'event_id', 'scheme', 'dst_ip', 'socket_ip', 'src_ip'}::ARRAY(STRING), {extra.uuid, transport.scheme, inet_to_string(socket.dst_ip), inet_to_string(socket.src_ip), inet_to_string(@src_ip::INET)}::ARRAY(STRING))") + clFSL += fsl.AppendInto(CommonLog, "batch_set_payload_int", "", "weblog_payload_set_int_batch({'src_port', 'dst_port'}::ARRAY(STRING), {socket.src_port, socket.dst_port}::ARRAY(INTEGER))") + clFSL += fsl.AppendInto(CommonLog, "batch_set_extra_bool", "", "weblog_extra_set_bool_batch({'alog_save_to_db', 'convert_to_blog', 'forward_log', 'convert_to_alog', 'convert_to_dlog', 'dlog_save_to_db', 'blog_save_to_db'}::ARRAY(STRING), {@alog_save_to_db::INTEGER, @convert_to_blog::INTEGER, @forward_log::INTEGER, @convert_to_alog::INTEGER, @convert_to_dlog::INTEGER, @dlog_save_to_db::INTEGER, @blog_save_to_db::INTEGER}::ARRAY(INTEGER))") + clFSL += fsl.AppendInto(CommonLog, "generate", "", "weblog_generate()") + return clFSL +} + +func ReqWeblogGenerateTable() (reqWGFSL string) { + reqWGFSL += fsl.CreateTable(ReqWeblogGenerate, nil) + reqWGFSL += fsl.AppendInto(ReqWeblogGenerate, "init_http_req_body", "", fsl.Actions(fsl.Set("@raw_req_body", STR, "http.raw_req_body"))) + reqWGFSL += fsl.AppendInto(ReqWeblogGenerate, "cut_http_req_body", "string.length(@raw_req_body::STRING) > 1048576", fsl.Actions(fsl.Set("@raw_req_body", STR, "string.substr(http.raw_req_body, 0, 1048576)"))) + reqWGFSL += fsl.AppendInto(ReqWeblogGenerate, "batch_set_payload_binary", "", "weblog_payload_set_binary_batch({'req_payload', 'req_body'}::ARRAY(STRING), {skynet.payload, @raw_req_body::STRING}::ARRAY(STRING))") + reqWGFSL += fsl.AppendInto(ReqWeblogGenerate, "batch_set_payload_string", "", "weblog_payload_set_string_batch({'site_uuid', 'req_header', 'req_detector_name', 'req_proxy_name', 'req_rule_id', 'req_location', 'req_decode_path', 'user_agent', 'query_string', 'method', 'host', 'referer', 'url_path', 'cookie'}::ARRAY(STRING), {@site_uuid::STRING, http.raw_req_header, fusion.node_id, extra.proxy_name, @rule_id::STRING, skynet.location, skynet.decode_path, http.user_agent, http.decoded_query, http.raw_method, @host::STRING, http.referer, http.path, http.raw_cookie}::ARRAY(STRING))") + reqWGFSL += fsl.AppendInto(ReqWeblogGenerate, "batch_set_payload_int", "", "weblog_payload_set_int_batch({'req_start_time', 'req_detect_time', 'req_block_reason', 'req_attack_type', 'req_risk_level', 'req_end_time', 'req_action'}::ARRAY(STRING), {extra.req_start_time, now.timestamp_us - @start_time::INTEGER, fusion.block_reason, @attack_type::INTEGER, @risk_level::INTEGER, extra.req_end_time, @final_action::INTEGER}::ARRAY(INTEGER))") + reqWGFSL += fsl.AppendInto(ReqWeblogGenerate, "common_log", "", fsl.Goto(CommonLog)) + return reqWGFSL +} + +func RspWeblogGenerateTable() (rspWGFSL string) { + rspWGFSL += fsl.CreateTable(RspWeblogGenerate, nil) + rspWGFSL += fsl.AppendInto(RspWeblogGenerate, "init_http_rsp_body", "", fsl.Actions(fsl.Set("@raw_rsp_body", STR, "http.raw_rsp_body"))) + rspWGFSL += fsl.AppendInto(RspWeblogGenerate, "cut_http_rsp_body", "string.length(@raw_rsp_body::STRING) > 1048576", fsl.Actions(fsl.Set("@raw_rsp_body", STR, "string.substr(http.raw_rsp_body, 0, 1048576)"))) + rspWGFSL += fsl.AppendInto(RspWeblogGenerate, "batch_set_payload_binary", "", "weblog_payload_set_binary_batch({'rsp_payload', 'rsp_body'}::ARRAY(STRING), {skynet.payload, @raw_rsp_body::STRING}::ARRAY(STRING))") + rspWGFSL += fsl.AppendInto(RspWeblogGenerate, "batch_set_payload_string", "", "weblog_payload_set_string_batch({'rsp_rule_id', 'rsp_location', 'rsp_decode_path', 'rsp_header', 'rsp_detector_name', 'rsp_proxy_name'}::ARRAY(STRING), {@rule_id::STRING, skynet.location, skynet.decode_path, http.raw_rsp_header, fusion.node_id, extra.proxy_name}::ARRAY(STRING))") + rspWGFSL += fsl.AppendInto(RspWeblogGenerate, "batch_set_payload_int", "", "weblog_payload_set_int_batch({'rsp_detect_time', 'status_code', 'rsp_start_time', 'rsp_end_time', 'rsp_action', 'rsp_attack_type', 'rsp_risk_level', 'rsp_block_reason'}::ARRAY(STRING), {now.timestamp_us - @start_time::INTEGER, http.status_code, extra.rsp_start_time, extra.rsp_end_time, @final_action::INTEGER, @attack_type::INTEGER, @risk_level::INTEGER, fusion.block_reason}::ARRAY(INTEGER))") + rspWGFSL += fsl.AppendInto(RspWeblogGenerate, "common_log", "", fsl.Goto(CommonLog)) + return rspWGFSL +} + +func WeblogGenerateTable() (wgFSL string) { + wgFSL += CommonLogTable() + wgFSL += ReqWeblogGenerateTable() + wgFSL += RspWeblogGenerateTable() + wgFSL += fsl.CreateTable(WeblogGenerate, nil) + wgFSL += fsl.AppendInto(WeblogGenerate, "req", "fusion.stage = 0", fsl.Goto(ReqWeblogGenerate)) + wgFSL += fsl.AppendInto(WeblogGenerate, "rsp", "fusion.stage = 1", fsl.Goto(RspWeblogGenerate)) + return wgFSL +} + +func MainTable() (mainFSL string) { + mainFSL += fsl.CreateTable(Main, nil) + mainFSL += fsl.AppendInto(Main, "m_preprocess", "", fsl.Goto(Preprocess)) + mainFSL += fsl.AppendInto(Main, "m_mass_package", "", fsl.Goto(MassPackage)) + mainFSL += fsl.AppendInto(Main, "m_policy_rule", "@skip_remaining::BOOLEAN = false", fsl.Goto(PolicyRule)) + mainFSL += fsl.AppendInto(Main, "m_policy_group", "@skip_remaining::BOOLEAN = false", fsl.Goto(PolicyGroup)) + mainFSL += fsl.AppendInto(Main, "m_weblog_generate", "", fsl.Goto(WeblogGenerate)) + mainFSL += "ENTRYPOINT TABLE main;" + return mainFSL +} diff --git a/management/webserver/pkg/fvm/helper.go b/management/webserver/pkg/fvm/helper.go new file mode 100644 index 0000000..9aba93b --- /dev/null +++ b/management/webserver/pkg/fvm/helper.go @@ -0,0 +1,56 @@ +package fvm + +import ( + "chaitin.cn/dev/go/errors" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" + "chaitin.cn/patronus/safeline-2/management/webserver/utils" +) + +var GlobalFVM *FVM + +func init() { + var err error + GlobalFVM, err = New() + if err != nil { + panic(err) + } +} + +func CompileAndSave(text string) error { + output, _, err := GlobalFVM.Compile(text, nil) + if err != nil { + return errors.Annotate(err, "failed to compile rules") + } + defer ReleaseOutput(output) + + update := Serialize(output, false, 0, 1) + defer update.Release() + + // save to file + if err = utils.EnsureWriteFile(config.GlobalConfig.Detector.FslBytecode, update.ToBytes(), 0666); err != nil { + return errors.Annotate(err, "failed to save bytecode") + } + + return nil +} + +func CompileAndPush(text, serverAddr string) error { + output, _, err := GlobalFVM.Compile(text, nil) + if err != nil { + return errors.Annotate(err, "failed to compile rules") + } + defer ReleaseOutput(output) + + update := Serialize(output, false, 0, 1) + defer update.Release() + + // save to file + if err = utils.EnsureWriteFile(config.GlobalConfig.Detector.FslBytecode, update.ToBytes(), 0666); err != nil { + return errors.Annotate(err, "failed to save bytecode") + } + + if err := GlobalFVM.PushFsl(serverAddr, update); err != nil { + return errors.Annotatef(err, "failed to push rule to '%s'", serverAddr) + } + return nil +} diff --git a/management/webserver/pkg/log/log.go b/management/webserver/pkg/log/log.go new file mode 100644 index 0000000..3af9a18 --- /dev/null +++ b/management/webserver/pkg/log/log.go @@ -0,0 +1,117 @@ +package log + +import ( + "fmt" + "os" + "runtime" + "strings" + + "github.com/sirupsen/logrus" + + "chaitin.cn/dev/go/log" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" + "chaitin.cn/patronus/safeline-2/management/webserver/utils" +) + +func GetLogger(name string) *log.Logger { + return log.GetLogger(name) +} + +func LoadLogLevel() { + lv, _ := log.ParseLevel(config.GlobalConfig.Log.Level) + log.SetLevel(log.AllLoggers, lv) +} + +func SetLogFormatter() { + // format + formatter := new(log.TextFormatter) + formatter.FullTimestamp = true + formatter.TimestampFormat = "2006/01/02 15:04:05" + log.SetFormatter(log.AllLoggers, formatter) +} + +func InitLogger() error { + // output + switch config.GlobalConfig.Log.Output { + case "stdout": + log.SetOutput(log.AllLoggers, os.Stdout) + case "stderr": + log.SetOutput(log.AllLoggers, os.Stderr) + default: + exist, err := utils.FileExist(config.GlobalConfig.Log.Output) + if err != nil { + return err + } + + fileFlag := os.O_WRONLY | os.O_APPEND | os.O_SYNC + if !exist { + if err := utils.EnsureFileDir(config.GlobalConfig.Log.Output); err != nil { + return err + } + fileFlag = fileFlag | os.O_CREATE + } + + if fp, err := os.OpenFile(config.GlobalConfig.Log.Output, fileFlag, os.ModePerm); err != nil { + return fmt.Errorf("failed to open log file: %s", err.Error()) + } else { + log.SetOutput(log.AllLoggers, log.NewLockOutput(fp)) + } + } + + // hook + log.AddHook(log.AllLoggers, NewRuntimeHook()) + log.AddHook(log.AllLoggers, log.NewErrorStackHook(true)) + // level + LoadLogLevel() + + return nil +} + +type RuntimeHook struct{} + +func (h *RuntimeHook) Levels() []logrus.Level { + return logrus.AllLevels +} + +func (h *RuntimeHook) Fire(entry *logrus.Entry) error { + file := "???" + funcName := "???" + line := 0 + + pc := make([]uintptr, 64) + // Skip runtime.Callers, self, and another call from logrus + n := runtime.Callers(3, pc) + if n != 0 { + pc = pc[:n] // pass only valid pcs to runtime.CallersFrames + frames := runtime.CallersFrames(pc) + + // Loop to get frames. + // A fixed number of pcs can expand to an indefinite number of Frames. + for { + frame, more := frames.Next() + if !strings.Contains(frame.File, "github.com/sirupsen/logrus") && !strings.Contains(frame.Function, "chaitin.cn/dev/go") { + file = frame.File + funcName = frame.Function + line = frame.Line + break + } + if !more { + break + } + } + } + + slices := strings.Split(file, "/") + file = slices[len(slices)-1] + + funcName = strings.ReplaceAll(funcName, "chaitin.cn", "") + + entry.Data["file"] = file + entry.Data["func"] = funcName + entry.Data["line"] = line + return nil +} + +func NewRuntimeHook() *RuntimeHook { + return &RuntimeHook{} +} diff --git a/management/webserver/pkg/telemetry.go b/management/webserver/pkg/telemetry.go new file mode 100644 index 0000000..8b310ff --- /dev/null +++ b/management/webserver/pkg/telemetry.go @@ -0,0 +1,76 @@ +package pkg + +import ( + "fmt" + "io" + "net/http" + "strconv" + "time" + + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" + "chaitin.cn/patronus/safeline-2/management/webserver/utils" +) + +type WebsiteResult struct { + Id uint + UpdatedAt time.Time +} + +type TelemetryInfo struct { + Id string `json:"id"` +} + +type SafelineInfo struct { + Id string `json:"id"` + + Type string `json:"type"` + Version string `json:"version"` + + ReqCnt int `json:"req_cnt"` + DetectLogCnt int `json:"detect_log_cnt"` + SiteCnt int `json:"site_cnt"` + HealthySiteCnt int `json:"healthy_site_cnt"` + RuleCnt int `json:"rule_cnt"` + IsHealthy bool `json:"is_healthy"` + + Behavior map[string]int `json:"behavior"` + Websites []WebsiteResult `json:"websites"` + WebsitesCnt int `json:"websites_cnt"` +} + +type TelemetryRequest struct { + Telemetry TelemetryInfo `json:"telemetry"` + Safeline SafelineInfo `json:"safeline"` +} + +func GetUploadTimestamp() string { + yesterday := time.Now().AddDate(0, 0, -1) + yesterdayNoon := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 12, 0, 0, 0, yesterday.Location()) + return strconv.FormatInt(yesterdayNoon.Unix(), 10) +} + +func GetUploadNonce() string { + now := strconv.FormatInt(time.Now().Unix(), 10) + randStr := utils.RandStr(6) + return fmt.Sprintf("%s%s", now, randStr) +} + +func DoPostTelemetry(client *http.Client, addr string, reader io.Reader) (*http.Response, error) { + if addr == "" { + return nil, nil + } + req, err := http.NewRequest(http.MethodPost, "https://"+addr+constants.TelemetryEntryPoint, reader) + if err != nil { + return nil, err + } + + req.Header.Set("accept", constants.ApplicationJson) + req.Header.Set("Content-Type", constants.ApplicationJson) + + rsp, err := client.Do(req) + if err != nil { + return nil, err + } + + return rsp, nil +} diff --git a/management/webserver/proto/website/website.proto b/management/webserver/proto/website/website.proto new file mode 100644 index 0000000..d272481 --- /dev/null +++ b/management/webserver/proto/website/website.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; +package website; +option go_package = "proto/website"; + +service Website { + rpc Subscribe(stream Response) returns (stream Event) {} +} + +// From client-side, may be "pong" +message Response { + string type = 1; + bytes msg = 2; + bool err = 3; +} + +// From server-side, may be "ping" +message Event { + string type = 1; // ping/website + bytes msg = 2; +} diff --git a/management/webserver/rpc/main.go b/management/webserver/rpc/main.go new file mode 100644 index 0000000..3b5ae8c --- /dev/null +++ b/management/webserver/rpc/main.go @@ -0,0 +1,44 @@ +package rpc + +import ( + "net" + "time" + + "google.golang.org/grpc" + + "chaitin.cn/dev/go/errors" + "chaitin.cn/dev/go/log" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" + pb "chaitin.cn/patronus/safeline-2/management/webserver/proto/website" +) + +const ( + WaitRspTimeout = 30 * time.Second + KeepaliveTime = 5 * time.Second + KeepaliveTimeout = 30 * time.Second + + Ping = "ping" + Pong = "pong" + EventTypeWebsite = "website" + EventTypeDeleteWebsite = "deleteWebsite" + EventTypeFullWebsite = "fullWebsite" +) + +var logger = log.GetLogger("grpc") + +func StartGRPCSever() error { + lis, err := net.Listen("tcp", config.GlobalConfig.GPRC.ListenAddr) + if err != nil { + return errors.Wrap(err, "Failed to listen") + } + var opts []grpc.ServerOption + grpcServer := grpc.NewServer(opts...) + pb.RegisterWebsiteServer(grpcServer, GetWebsiteServer()) + go func() { + err := grpcServer.Serve(lis) + if err != nil { + logger.Fatalln("Failed to server") + } + }() + return nil +} diff --git a/management/webserver/rpc/website.go b/management/webserver/rpc/website.go new file mode 100644 index 0000000..0801292 --- /dev/null +++ b/management/webserver/rpc/website.go @@ -0,0 +1,182 @@ +package rpc + +import ( + "encoding/json" + "io" + "time" + + "chaitin.cn/dev/go/errors" + "chaitin.cn/patronus/safeline-2/management/webserver/model" + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" + pb "chaitin.cn/patronus/safeline-2/management/webserver/proto/website" +) + +var ( + ping = pb.Event{Type: Ping, Msg: nil} + Subscriber *StreamClient // only ONE sub +) + +func Publish(msg []byte, eventType string) error { + if Subscriber == nil { + return errors.New("Service is abnormal, and the nginx conf cannot be updated for the time being. Please go to the shell to check the relevant logs") + } + err := Subscriber.stream.Send(&pb.Event{ + Type: eventType, + Msg: msg, + }) + if err != nil { + select { + case <-Subscriber.stream.Context().Done(): + err = Subscriber.stream.Context().Err() // context canceled + Subscriber = nil + return err + case Subscriber.errCh <- errors.Wrapf(err, "Send event err: %s", msg): + return errors.Wrapf(err, "Send event err: %s", msg) + } + } + + rspTimeoutTicker := time.NewTicker(WaitRspTimeout) + defer rspTimeoutTicker.Stop() + + for { + select { + case rsp := <-Subscriber.rspCh: + if rsp.Err { + return errors.New(string(rsp.Msg)) + } else { + // success + return nil + } + case <-rspTimeoutTicker.C: + return errors.New("Wait timeout for updating result") + } + } +} + +// StreamClient is the instance for every client stream connected +type StreamClient struct { + stream pb.Website_SubscribeServer + timer *time.Timer // timer for timeout + errCh chan error + rspCh chan *pb.Response + quit chan struct{} // quit stream client gracefully +} + +func newStreamClient(stream pb.Website_SubscribeServer) *StreamClient { + return &StreamClient{ + stream: stream, + timer: time.NewTimer(KeepaliveTimeout), + errCh: make(chan error, 1), + rspCh: make(chan *pb.Response, 1), + quit: make(chan struct{}), + } +} + +func (sc *StreamClient) pingLoop() { + pingTicker := time.NewTicker(KeepaliveTime) + defer pingTicker.Stop() + + for { + if Subscriber != nil && sc != Subscriber { + // new subscriber in replace of the old one. + logger.Debug("New subscriber in replace of the old one") + return + } + + select { + case <-sc.stream.Context().Done(): + return + case <-pingTicker.C: + err := sc.stream.Send(&ping) + if err != nil { + select { + case <-sc.stream.Context().Done(): + case sc.errCh <- errors.Wrapf(err, "Send ping err"): + } + return + } + } + } +} + +func (sc *StreamClient) recvLoop() { + for { + if Subscriber != nil && sc != Subscriber { + // new subscriber in replace of the old one. + close(sc.quit) + logger.Debug("New subscriber in replace of the old one") + return + } + + rsp, err := sc.stream.Recv() + if err != nil { + if err != io.EOF { + select { + case <-sc.stream.Context().Done(): + case sc.errCh <- errors.Wrapf(err, "Recv pong err"): + } + } + return + } + + logger.Debugf("Got message Type %s", rsp.GetType()) + if rsp.Type != Pong { + // receive response + logger.Infof("Recv updating website rsp: err(%t), msg(%s)", rsp.GetErr(), rsp.GetMsg()) + sc.rspCh <- rsp + continue + } + sc.timer.Reset(KeepaliveTimeout) + } +} + +// WebsiteServer is the gRPC server implementation +type WebsiteServer struct { + *pb.UnimplementedWebsiteServer +} + +func GetWebsiteServer() *WebsiteServer { + return &WebsiteServer{} +} + +func publishFullWebsite() error { + var websites []model.Website + db := database.GetDB().DB + db.Model(&model.Website{}).Find(&websites) + byteWebsites, err := json.Marshal(&websites) + if err != nil { + return err + } + + return Publish(byteWebsites, EventTypeFullWebsite) +} + +// Subscribe is gRPC API entrypoint +func (ws *WebsiteServer) Subscribe(stream pb.Website_SubscribeServer) error { + Subscriber = newStreamClient(stream) + defer Subscriber.timer.Stop() + + go Subscriber.pingLoop() + go Subscriber.recvLoop() + + if err := publishFullWebsite(); err != nil { + // triggered when tcd starts, ignore push error messages + logger.Warn(err) + } + + select { + case <-Subscriber.quit: + logger.Infof("Disconnected gracefully") + return nil + case <-Subscriber.timer.C: + logger.Error("Keepalive timeout") + case err := <-Subscriber.errCh: + logger.WithError(err).Error() + return err + case <-Subscriber.stream.Context().Done(): + logger.Infof("Subscribe context done: %s", Subscriber.stream.Context().Err()) + Subscriber = nil + } + + return nil +} diff --git a/management/webserver/tools/init_db.sh b/management/webserver/tools/init_db.sh new file mode 100644 index 0000000..9c54ab6 --- /dev/null +++ b/management/webserver/tools/init_db.sh @@ -0,0 +1,4 @@ +docker rm -f mgt-postgres-dev + +#docker run -it -d -e POSTGRES_DB=safeline-ce -e POSTGRES_USER=safeline-ce -e POSTGRES_PASSWORD=safeline-ce -p 127.0.0.1:5432:5432 --name mgt-postgres-dev chaitin.cn/library/postgres:15.2 +docker run -it -d -e POSTGRES_DB=safeline-ce -e POSTGRES_USER=safeline-ce -e POSTGRES_PASSWORD=safeline-ce -p 127.0.0.1:5432:5432 --name mgt-postgres-dev postgres:15.2 \ No newline at end of file diff --git a/management/webserver/utils/cert.go b/management/webserver/utils/cert.go new file mode 100644 index 0000000..816cd0c --- /dev/null +++ b/management/webserver/utils/cert.go @@ -0,0 +1,109 @@ +package utils + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "io/ioutil" + "math/big" + "time" +) + +func GenerateCert(hostnames []string, days int64, keyBits int, subject *pkix.Name, isCA bool) ([]byte, []byte, error) { + privateKey, err := rsa.GenerateKey(rand.Reader, keyBits) + if err != nil { + return nil, nil, err + } + + notBefore := time.Now() + duration := int64(time.Hour) * 24 * days + notAfter := notBefore.Add(time.Duration(duration)) + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return nil, nil, err + } + + var keyUsage x509.KeyUsage + if isCA { + keyUsage = x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign + } else { + keyUsage = x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment + } + + template := x509.Certificate{ + SerialNumber: serialNumber, + NotBefore: notBefore, + NotAfter: notAfter, + DNSNames: hostnames, + BasicConstraintsValid: true, + IsCA: isCA, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + KeyUsage: keyUsage, + } + if subject != nil { + template.Subject = *subject + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + if err != nil { + return nil, nil, err + } + + var certBuffer bytes.Buffer + err = pem.Encode(&certBuffer, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + if err != nil { + return nil, nil, err + } + + var keyBuffer bytes.Buffer + err = pem.Encode(&keyBuffer, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)}) + if err != nil { + return nil, nil, err + } + + return certBuffer.Bytes(), keyBuffer.Bytes(), nil +} + +func WriteCertIfNotExist(certFilePath, keyFilePath string, generator func() ([]byte, []byte, error)) error { + cert, key, err := genCertIfNotExist(certFilePath, keyFilePath, generator) + if err != nil { + return err + } + + if err = EnsureRenameWriteFile(certFilePath, cert, 0644); err != nil { + return err + } + + if err = EnsureRenameWriteFile(keyFilePath, key, 0644); err != nil { + return err + } + + return nil +} + +func genCertIfNotExist(certFilePath, keyFilePath string, generator func() ([]byte, []byte, error)) ([]byte, []byte, error) { + exist, err := FilesExist(certFilePath, keyFilePath) + if err != nil { + return nil, nil, err + } + + if !exist { + return generator() + } else { + cert, err := ioutil.ReadFile(certFilePath) + if err != nil { + return nil, nil, err + } + + key, err := ioutil.ReadFile(keyFilePath) + if err != nil { + return nil, nil, err + } + return cert, key, nil + } +} diff --git a/management/webserver/utils/file.go b/management/webserver/utils/file.go new file mode 100644 index 0000000..4ae2d88 --- /dev/null +++ b/management/webserver/utils/file.go @@ -0,0 +1,75 @@ +package utils + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + +func EnsureDir(dir string) error { + if _, err := os.Stat(dir); os.IsNotExist(err) { + return os.MkdirAll(dir, os.FileMode(0755)) + } + return nil +} + +func EnsureFileDir(path string) error { + return EnsureDir(filepath.Dir(path)) +} + +func FileExist(path string) (bool, error) { + stat, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } else { + return false, err + } + } else { + if stat.IsDir() { + return false, fmt.Errorf("%s is dir", path) + } else { + return true, nil + } + } +} + +func FilesExist(paths ...string) (bool, error) { + for _, path := range paths { + exist, err := FileExist(path) + if err != nil { + return false, err + } + if !exist { + return false, nil + } + } + return true, nil +} + +func RenameWriteFile(filename string, data []byte, perm os.FileMode) error { + randFileName := filename + ".tmp." + RandStr(8) + if err := ioutil.WriteFile(randFileName, data, perm); err != nil { + return err + } + return os.Rename(randFileName, filename) +} + +func EnsureRenameWriteFile(path string, data []byte, mode os.FileMode) error { + err := EnsureFileDir(path) + if err != nil { + return err + } + + return RenameWriteFile(path, data, mode) +} + +func EnsureWriteFile(path string, data []byte, mode os.FileMode) error { + err := EnsureFileDir(path) + if err != nil { + return err + } + + return ioutil.WriteFile(path, data, mode) +} diff --git a/management/webserver/utils/healthy.go b/management/webserver/utils/healthy.go new file mode 100644 index 0000000..ef64343 --- /dev/null +++ b/management/webserver/utils/healthy.go @@ -0,0 +1,31 @@ +package utils + +import ( + "fmt" + "net/http" +) + +func CheckHealthy(url string) bool { + method := "GET" + + client := &http.Client{} + req, err := http.NewRequest(method, url, nil) + + if err != nil { + fmt.Println(err) + return false + } + res, err := client.Do(req) + if err != nil { + fmt.Println(err) + return false + } + defer res.Body.Close() + + return res.StatusCode == 200 +} + +func CheckWafHealthy() bool { + // todo: not check the healthy status + return CheckHealthy("http://safeline-mario:3335") && CheckHealthy("http://safeline-detector:8001/stat") +} diff --git a/management/webserver/utils/httpclient.go b/management/webserver/utils/httpclient.go new file mode 100644 index 0000000..d3874d0 --- /dev/null +++ b/management/webserver/utils/httpclient.go @@ -0,0 +1,35 @@ +package utils + +import ( + "crypto/tls" + "net/http" + "net/url" + "os" + "time" +) + +const proxyName = "HTTPS_PROXY" + +var httpClient *http.Client + +func GetHTTPClient() *http.Client { + if httpClient == nil { + tr := &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 30 * time.Second, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + + proxyUrl, existed := os.LookupEnv(proxyName) + if existed { + uri, _ := url.Parse(proxyUrl) + tr.Proxy = http.ProxyURL(uri) + } + + httpClient = &http.Client{Transport: tr} + } + + return httpClient +} diff --git a/management/webserver/utils/random.go b/management/webserver/utils/random.go new file mode 100644 index 0000000..44ebd98 --- /dev/null +++ b/management/webserver/utils/random.go @@ -0,0 +1,18 @@ +package utils + +import ( + "math/rand" + "time" +) + +var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + +func RandStr(n int) string { + rand.Seed(time.Now().UnixNano()) + + b := make([]rune, n) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) +} diff --git a/management/webserver/utils/url.go b/management/webserver/utils/url.go new file mode 100644 index 0000000..84b633f --- /dev/null +++ b/management/webserver/utils/url.go @@ -0,0 +1,34 @@ +package utils + +import ( + "fmt" + "net" + "strings" + + "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" +) + +func IsIPv6(str string) bool { + ip := net.ParseIP(str) + return ip != nil && strings.Contains(str, ":") +} + +func BuildUrl(protocol int, host string, port uint, path string) string { + portStr := "" + if len(host) > 0 && IsIPv6(host) && host[:1] != "[" { + host = fmt.Sprintf("[%s]", host) + } + if protocol == constants.ProtocolHTTP { + if port != 80 { + portStr = fmt.Sprintf(":%d", port) + } + } else if protocol == constants.ProtocolHTTPS || protocol == constants.ProtocolHTTP2 { + if port != 443 { + portStr = fmt.Sprintf(":%d", port) + } + } else { + // use HTTP as default protocol + protocol = constants.ProtocolHTTP + } + return fmt.Sprintf("%s://%s%s%s", constants.HTTPProtocol[protocol], host, portStr, path) +} diff --git a/mcp_server/Dockerfile b/mcp_server/Dockerfile new file mode 100644 index 0000000..3d96634 --- /dev/null +++ b/mcp_server/Dockerfile @@ -0,0 +1,28 @@ +FROM golang:1.24-alpine AS builder + +WORKDIR /app + +RUN apk add --no-cache git + +COPY go.mod go.sum ./ + +RUN go mod download + +COPY . . + +RUN CGO_ENABLED=0 go build -o mcp-server . + +FROM alpine:latest + +WORKDIR /app + +RUN apk add --no-cache ca-certificates tzdata + +COPY --from=builder /app/mcp-server . +COPY --from=builder /app/config.yaml . + +ENV TZ=Asia/Shanghai + +EXPOSE 5678 + +CMD ["./mcp-server"] \ No newline at end of file diff --git a/mcp_server/README.md b/mcp_server/README.md new file mode 100644 index 0000000..b88b1c4 --- /dev/null +++ b/mcp_server/README.md @@ -0,0 +1,256 @@ +# SafeLine MCP Server + +SafeLine MCP Server is an implementation of the [Model Context Protocol (MCP)](https://modelcontextprotocol.io/introduction) that provides complete management and control capabilities for SafeLine WAF. + +[![Docker](https://img.shields.io/badge/Docker-Supported-2496ED?style=flat-square&logo=docker&logoColor=white)](docker-compose.yml) +[![Go Version](https://img.shields.io/badge/Go-1.21+-00ADD8?style=flat-square&logo=go&logoColor=white)](go.mod) + +## Use Cases + +- Automated management and control of SafeLine WAF instances +- WAF configuration and policy management through API +- Building AI-based security protection tools and applications + +## Prerequisites + +1. Install [Docker](https://www.docker.com/) (if running in container) +2. Configure SafeLine API Token (obtained from SafeLine console) + +## Features + +- Complete MCP (Management Control Protocol) server implementation +- Support for SafeLine WAF instance management and control +- Flexible configuration system supporting file configuration and environment variables +- Docker containerization support +- Secure API communication + +## Quick Start + +### Environment Variables + +| Environment Variable | Description | Default Value | Required | +|---------|------|--------|-----| +| LISTEN_PORT | Service listening port | 5678 | No | +| LISTEN_ADDRESS | Service listening address | 0.0.0.0 | No | +| SAFELINE_SECRET | SSE server secret | - | No | +| SAFELINE_ADDRESS | SafeLine API address | - | Yes | +| SAFELINE_API_TOKEN | SafeLine API authentication token | - | Yes | + +### Using Docker + +#### Method 1: Using docker run + +```bash +docker run -d \ + --name safeline-mcp \ + -p 5678:5678 \ + -e SAFELINE_API_TOKEN="your_api_token" \ + -e SAFELINE_ADDRESS="https://your.safeline.com" \ + -e LISTEN_PORT=5678 \ + -e LISTEN_ADDRESS="0.0.0.0" \ + chaitin/safeline-mcp:latest +``` + +#### Method 2: Using docker-compose + +```bash +# 1. Clone repository +git clone https://github.com/chaitin/safeline-mcp.git +cd safeline-mcp + +# 2. Edit docker-compose.yml to configure environment variables +# Example docker-compose.yml: +# version: '3' +# services: +# mcp: +# image: chaitin/safeline-mcp:latest +# container_name: safeline-mcp +# ports: +# - "5678:5678" +# environment: +# - SAFELINE_API_TOKEN=your_api_token +# - SAFELINE_ADDRESS=https://your.safeline.com +# - LISTEN_PORT=5678 +# - LISTEN_ADDRESS=0.0.0.0 + +# 3. Start service +docker compose -f docker-compose.yml up -d +``` + +#### Method 3: Using Go + +```bash +# 1. Clone repository +git clone https://github.com/chaitin/SafeLine.git +cd safeline-mcp + +# 2. Install dependencies +go mod download + +# 3. Configure config.yaml +cp config.yaml.example config.yaml +# Edit config.yaml with necessary configurations + +# 4. Run service +go run main.go +``` + +For more API details, please refer to the [API Documentation](https://demo.waf.chaitin.com:9443/swagger/index.html). + +## Tools + +### Application Management + +- **create_application** + +### Rule Management +- **create_blacklist_rule** +- **create_whitelist_rule** + +### Analyze +- **get_attack_events** + +## Development Guide + +The Go API in this project is currently under development, and APIs may change. If you have specific requirements, please submit an Issue for discussion. + +### Directory Structure + +``` +internal/ +├── api/ # API implementation +│ ├── app/ # Application-related APIs +│ │ └── create_application.go +│ └── rule/ # Rule-related APIs +│ └── create_rule.go +└── tools/ # MCP tool implementation + ├── app/ # Application-related tools + │ └── create_application.go + └── rule/ # Rule-related tools + └── create_rule.go +``` + +### Adding New Tools + +1. **Create Tool File** + - Create corresponding directory and file under `internal/tools` + - File name should match tool name + - Use separate file for each tool + - Example: `internal/tools/app/create_application.go` + +2. **Tool Implementation Template** +```go +package app + +type ToolName struct{} + +type ToolParams struct { + // Parameter definitions + Param1 string `json:"param1" desc:"parameter description" required:"true"` + Param2 int `json:"param2" desc:"parameter description" required:"false"` +} + +type ToolResult struct { + Field1 string `json:"field1"` +} + +func (t *ToolName) Name() string { + return "tool_name" +} + +func (t *ToolName) Description() string { + return "tool description" +} + +func (t *ToolName) Validate(params ToolParams) error { + // Parameter validation logic + return nil +} + +func (t *ToolName) Execute(ctx context.Context, params ToolParams) (result ToolResult, err error) { + // Tool execution logic + return result, nil +} +``` + +3. **[Optional]Create API Implementation** + +If you need to use some APIs that have not been implemented yet, you need to create corresponding files in the api directory for implementation + - Create same directory structure under `internal/api` + - File name should match tool func + - Example: `internal/api/app/create_application.go` + +**API Implementation Template** +```go +package app + +type RequestType struct { + // Request parameter definitions + Param1 string `json:"param1"` + Param2 int `json:"param2"` +} + +func APIName(ctx context.Context, req *RequestType) (ResultType, error) { + if req == nil { + return nil, errors.New("request is required") + } + + var resp api.Response[ResultType] + err := api.Service().Post(ctx, "/api/path", req, &resp) + if err != nil { + return nil, errors.Wrap(err, "failed to execute") + } + + if resp.Err != nil { + return nil, errors.New(resp.Msg) + } + + return resp.Data, nil +} +``` +4. **Tool Registration (init.go)** + +The tool registration file `internal/tools/init.go` is used to centrally manage all tool registrations + - Register all tools uniformly in the `init()` function + - Use the `AppendTool()` method for registration + - Example: + ```go + // Register create application tool + AppendTool(&app.CreateApp{}) + + // Register create blacklist rule tool + AppendTool(&rule.CreateBlacklistRule{}) + ``` + +### Development Standards + +1. **Naming Conventions** + - Use lowercase letters and underscores for tool names + - File names should match tool names + +2. **Directory Organization** + - Divide directories by functional modules (e.g., app, rule, etc.) + - Maintain consistent structure between tools and api directories + - Keep related functionality in the same directory + +3. **Code Standards** + - Follow Go standard code conventions + - Add necessary parameter validation + - Use unified error handling approach + - Add appropriate logging + +4. **Documentation Requirements** + - Provide clear functional description in tool Description + - Add detailed description for parameters + - Update API toolkit documentation in README + +### Example + +Refer to the implementation of the `create_application` tool: +- Tool implementation: `internal/tools/app/create_application.go` +- API implementation: `internal/api/app/create_application.go` + + + + + diff --git a/mcp_server/config.yaml b/mcp_server/config.yaml new file mode 100644 index 0000000..87b5b78 --- /dev/null +++ b/mcp_server/config.yaml @@ -0,0 +1,27 @@ +# Server Configuration +server: + name: "SafeLine MCP Server" + version: "1.0.0" + # Can be overridden by environment variable LISTEN_PORT + port: 5678 + # Can be overridden by environment variable LISTEN_ADDRESS + host: "0.0.0.0" + # Can be overridden by environment variable SAFELINE_SECRET + secret: "" # Secret for SSE server +# Logger Configuration +logger: + level: "info" # Log level: debug, info, warn, error + file_path: "" # Log file path + console: true # Whether to output to console + caller: false # Whether to record caller information + development: true # Whether to use development mode + +# API Configuration +api: + # Can be overridden by environment variable SAFELINE_ADDRESS + base_url: "" # API service address + # Can be overridden by environment variable SAFELINE_API_TOKEN + token: "" # Authentication token + timeout: 30 # Timeout in seconds + debug: false # Whether to enable debug mode + insecure_skip_verify: true # Whether to skip certificate verification diff --git a/mcp_server/docker-compose.yml b/mcp_server/docker-compose.yml new file mode 100644 index 0000000..2ffdafe --- /dev/null +++ b/mcp_server/docker-compose.yml @@ -0,0 +1,15 @@ +version: '3.8' + +services: + mcp_server: + image: chaitin/safeline-mcp:latest + container_name: mcp_server + restart: always + ports: + - "5678:5678" + environment: + - SAFELINE_SECRET=your_secret_key # optional, if you want to use secret key to authenticate + - SAFELINE_ADDRESS=https://your_safeline_ip:9443 # required, your SafeLine WAF address + - SAFELINE_API_TOKEN=your_safeline_api_token # required, your SafeLine WAF api token + - LISTEN_PORT=5678 # optional, default is 5678 + - LISTEN_ADDRESS=0.0.0.0 # optional, default is 0.0.0.0 \ No newline at end of file diff --git a/mcp_server/go.mod b/mcp_server/go.mod new file mode 100644 index 0000000..7b83623 --- /dev/null +++ b/mcp_server/go.mod @@ -0,0 +1,16 @@ +module github.com/chaitin/SafeLine/mcp_server + +go 1.24.1 + +require ( + github.com/mark3labs/mcp-go v0.18.0 + go.uber.org/zap v1.27.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/google/uuid v1.6.0 // indirect + github.com/mcuadros/go-defaults v1.2.0 + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + go.uber.org/multierr v1.11.0 // indirect +) diff --git a/mcp_server/go.sum b/mcp_server/go.sum new file mode 100644 index 0000000..5c31060 --- /dev/null +++ b/mcp_server/go.sum @@ -0,0 +1,28 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mark3labs/mcp-go v0.18.0 h1:YuhgIVjNlTG2ZOwmrkORWyPTp0dz1opPEqvsPtySXao= +github.com/mark3labs/mcp-go v0.18.0/go.mod h1:KmJndYv7GIgcPVwEKJjNcbhVQ+hJGJhrCCB/9xITzpE= +github.com/mcuadros/go-defaults v1.2.0 h1:FODb8WSf0uGaY8elWJAkoLL0Ri6AlZ1bFlenk56oZtc= +github.com/mcuadros/go-defaults v1.2.0/go.mod h1:WEZtHEVIGYVDqkKSWBdWKUVdRyKlMfulPaGDWIVeCWY= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/mcp_server/internal/api/analyze/get_event_list.go b/mcp_server/internal/api/analyze/get_event_list.go new file mode 100644 index 0000000..29eac72 --- /dev/null +++ b/mcp_server/internal/api/analyze/get_event_list.go @@ -0,0 +1,47 @@ +package analyze + +import ( + "context" + "fmt" + + "github.com/chaitin/SafeLine/mcp_server/internal/api" +) + +type GetEventListRequest struct { + Page int `json:"page"` + PageSize int `json:"page_size"` + IP string `json:"ip"` + Start int64 `json:"start"` + End int64 `json:"end"` +} + +type GetEventListResponse struct { + Nodes []Event `json:"nodes"` + Total int64 `json:"total"` +} + +type Event struct { + ID uint `json:"id"` + IP string `json:"ip"` + Protocol int `json:"protocol"` + Host string `json:"host"` + DstPort uint64 `json:"dst_port"` + UpdatedAt int64 `json:"updated_at"` + StartAt int64 `json:"start_at"` + EndAt int64 `json:"end_at"` + DenyCount int64 `json:"deny_count"` + PassCount int64 `json:"pass_count"` + Finished bool `json:"finished"` + Country string `json:"country"` + Province string `json:"province"` + City string `json:"city"` +} + +func GetEventList(ctx context.Context, req *GetEventListRequest) (*GetEventListResponse, error) { + var resp api.Response[GetEventListResponse] + err := api.Service().Get(ctx, fmt.Sprintf("/api/open/events?page=%d&page_size=%d&ip=%s&start=%d&end=%d", req.Page, req.PageSize, req.IP, req.Start, req.End), &resp) + if err != nil { + return nil, err + } + return &resp.Data, nil +} diff --git a/mcp_server/internal/api/app/create_application.go b/mcp_server/internal/api/app/create_application.go new file mode 100644 index 0000000..3d7d041 --- /dev/null +++ b/mcp_server/internal/api/app/create_application.go @@ -0,0 +1,34 @@ +package app + +import ( + "context" + + "github.com/chaitin/SafeLine/mcp_server/internal/api" + "github.com/chaitin/SafeLine/mcp_server/pkg/errors" +) + +type CreateAppRequest struct { + ServerNames []string `json:"server_names"` + Ports []string `json:"ports"` + Upstreams []string `json:"upstreams"` + Comment string `json:"comment"` +} + +// CreateApp Create new website or app +func CreateApp(ctx context.Context, req *CreateAppRequest) (int64, error) { + if req == nil { + return 0, errors.New("request is required") + } + + var resp api.Response[int64] + err := api.Service().Post(ctx, "/api/open/site", req, &resp) + if err != nil { + return 0, errors.Wrap(err, "failed to create app") + } + + if resp.Err != nil { + return 0, errors.New(resp.Msg) + } + + return resp.Data, nil +} diff --git a/mcp_server/internal/api/client.go b/mcp_server/internal/api/client.go new file mode 100644 index 0000000..7978172 --- /dev/null +++ b/mcp_server/internal/api/client.go @@ -0,0 +1,157 @@ +package api + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/chaitin/SafeLine/mcp_server/pkg/errors" + "github.com/chaitin/SafeLine/mcp_server/pkg/logger" +) + +// Client API client +type Client struct { + baseURL string + httpClient *http.Client + headers map[string]string +} + +// ClientOption Client configuration options +type ClientOption func(*Client) + +// WithTimeout Set timeout duration +func WithTimeout(timeout time.Duration) ClientOption { + return func(c *Client) { + c.httpClient.Timeout = timeout + } +} + +// WithHeader Set request header +func WithHeader(key, value string) ClientOption { + return func(c *Client) { + c.headers[key] = value + } +} + +// WithBaseURL Set base URL +func WithBaseURL(baseURL string) ClientOption { + return func(c *Client) { + c.baseURL = baseURL + } +} + +// WithInsecureSkipVerify Set whether to skip certificate verification +func WithInsecureSkipVerify(skip bool) ClientOption { + return func(c *Client) { + if transport, ok := c.httpClient.Transport.(*http.Transport); ok { + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + transport.TLSClientConfig.InsecureSkipVerify = skip + } + } +} + +// NewClient Create new API client +func NewClient(opts ...ClientOption) *Client { + transport := &http.Transport{ + TLSClientConfig: &tls.Config{}, + } + + c := &Client{ + httpClient: &http.Client{ + Timeout: 30 * time.Second, + Transport: transport, + }, + headers: make(map[string]string), + } + + for _, opt := range opts { + opt(c) + } + + return c +} + +// Request Send request +func (c *Client) Request(ctx context.Context, method, path string, body interface{}, result interface{}) error { + reqURL := fmt.Sprintf("%s%s", c.baseURL, path) + + var bodyReader io.Reader + if body != nil { + bodyBytes, err := json.Marshal(body) + if err != nil { + return errors.Wrap(err, "marshal request body failed") + } + bodyReader = bytes.NewReader(bodyBytes) + } + logger.With("url", reqURL).Debug("request url") + req, err := http.NewRequestWithContext(ctx, method, reqURL, bodyReader) + if err != nil { + return errors.Wrap(err, "create request failed") + } + + // Set common headers + req.Header.Set("Content-Type", "application/json") + for k, v := range c.headers { + req.Header.Set(k, v) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return errors.Wrap(err, "send request failed") + } + defer resp.Body.Close() + + // Read response body + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "read response body failed") + } + + // Check status code + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return errors.New(fmt.Sprintf("request failed with status %d: %s", resp.StatusCode, string(respBody))) + } + + // Parse response + if result != nil { + if err := json.Unmarshal(respBody, result); err == nil { + return nil + } + var respData map[string]interface{} + if err := json.Unmarshal(respBody, &respData); err != nil { + return errors.Wrap(err, "unmarshal response failed") + } + if respData["err"] != nil || respData["msg"] != nil { + return errors.New(respData["msg"].(string)) + } + } + + return nil +} + +// Get Send GET request +func (c *Client) Get(ctx context.Context, path string, result interface{}) error { + return c.Request(ctx, http.MethodGet, path, nil, result) +} + +// Post Send POST request +func (c *Client) Post(ctx context.Context, path string, body interface{}, result interface{}) error { + return c.Request(ctx, http.MethodPost, path, body, result) +} + +// Put Send PUT request +func (c *Client) Put(ctx context.Context, path string, body interface{}, result interface{}) error { + return c.Request(ctx, http.MethodPut, path, body, result) +} + +// Delete Send DELETE request +func (c *Client) Delete(ctx context.Context, path string, result interface{}) error { + return c.Request(ctx, http.MethodDelete, path, nil, result) +} diff --git a/mcp_server/internal/api/response.go b/mcp_server/internal/api/response.go new file mode 100644 index 0000000..bfa2993 --- /dev/null +++ b/mcp_server/internal/api/response.go @@ -0,0 +1,11 @@ +package api + +// Response Common API response structure +type Response[T any] struct { + // Response data + Data T `json:"data"` + // Error message + Err any `json:"err"` + // Prompt message + Msg string `json:"msg"` +} diff --git a/mcp_server/internal/api/rule/create_rule.go b/mcp_server/internal/api/rule/create_rule.go new file mode 100644 index 0000000..6eb8843 --- /dev/null +++ b/mcp_server/internal/api/rule/create_rule.go @@ -0,0 +1,35 @@ +package rule + +import ( + "context" + + "github.com/chaitin/SafeLine/mcp_server/internal/api" + "github.com/chaitin/SafeLine/mcp_server/pkg/errors" +) + +type CreateRuleRequest struct { + Name string `json:"name"` + IP []string `json:"ip"` + IsEnabled bool `json:"is_enabled"` + Pattern [][]api.Pattern `json:"pattern"` + Action int `json:"action"` +} + +// CreateRule Create new rule +func CreateRule(ctx context.Context, req *CreateRuleRequest) (int64, error) { + if req == nil { + return 0, errors.New("request is required") + } + + var resp api.Response[int64] + err := api.Service().Post(ctx, "/api/open/policy", req, &resp) + if err != nil { + return 0, errors.Wrap(err, "failed to create policy rule") + } + + if resp.Err != nil { + return 0, errors.New(resp.Msg) + } + + return resp.Data, nil +} diff --git a/mcp_server/internal/api/service.go b/mcp_server/internal/api/service.go new file mode 100644 index 0000000..2c0a62a --- /dev/null +++ b/mcp_server/internal/api/service.go @@ -0,0 +1,100 @@ +package api + +import ( + "context" + "sync" + "time" + + "github.com/chaitin/SafeLine/mcp_server/internal/config" + "github.com/chaitin/SafeLine/mcp_server/pkg/errors" + "github.com/chaitin/SafeLine/mcp_server/pkg/logger" +) + +// APIClient API client implementation +type APIClient struct { + client *Client + config *config.APIConfig +} + +var ( + instance *APIClient + once sync.Once +) + +// Init Initialize API service +func Init(cfg *config.APIConfig) error { + var err error + once.Do(func() { + instance, err = newAPIClient(cfg) + if err != nil { + logger.With("error", err).Error("failed to initialize API service") + return + } + logger.Info("API service initialized successfully") + }) + return err +} + +// Service Get API service instance +func Service() *APIClient { + if instance == nil { + logger.Error("API service not initialized") + panic("API service not initialized") + } + return instance +} + +// newAPIClient Create new API client +func newAPIClient(config *config.APIConfig) (*APIClient, error) { + if config == nil { + return nil, errors.New("config is required") + } + + if config.BaseURL == "" { + return nil, errors.New("base_url is required") + } + + timeout := 30 + if config.Timeout > 0 { + timeout = config.Timeout + } + + opts := []ClientOption{ + WithBaseURL(config.BaseURL), + WithTimeout(time.Duration(timeout) * time.Second), + WithHeader("User-Agent", "SafeLine-MCP/1.0"), + WithInsecureSkipVerify(config.InsecureSkipVerify), + } + + // If token is configured, add authentication header + if config.Token != "" { + opts = append(opts, WithHeader("X-SLCE-API-TOKEN", config.Token)) + } + + client := NewClient(opts...) + + return &APIClient{ + client: client, + config: config, + }, nil +} + +// Post Send POST request +func (c *APIClient) Post(ctx context.Context, path string, body interface{}, result interface{}) error { + return c.client.Request(ctx, "POST", path, body, result) +} + +// Get Send GET request +func (c *APIClient) Get(ctx context.Context, path string, result interface{}) error { + return c.client.Request(ctx, "GET", path, nil, result) +} + +// Put Send PUT request +func (c *APIClient) Put(ctx context.Context, path string, body interface{}, result interface{}) error { + return c.client.Request(ctx, "PUT", path, body, result) +} + +// Delete Send DELETE request +func (c *APIClient) Delete(ctx context.Context, path string, result interface{}) error { + return c.client.Request(ctx, "DELETE", path, nil, result) +} diff --git a/mcp_server/internal/api/types.go b/mcp_server/internal/api/types.go new file mode 100644 index 0000000..5113ce2 --- /dev/null +++ b/mcp_server/internal/api/types.go @@ -0,0 +1,50 @@ +package api + +type PolicyRuleAction int + +const ( + PolicyRuleActionAllow PolicyRuleAction = iota + PolicyRuleActionDeny + PolicyRuleActionMax +) + +type Key = string + +const ( + KeySrcIP Key = "src_ip" + KeyURI Key = "uri" + KeyURINoQuery Key = "uri_no_query" + KeyHost Key = "host" + KeyMethod Key = "method" + KeyReqHeader Key = "req_header" + KeyReqBody Key = "req_body" + KeyGetParam Key = "get_param" + KeyPostParam Key = "post_param" +) + +type Op = string + +const ( + OpEq Op = "eq" // equal + OpNotEq Op = "not_eq" // not equal + OpMatch Op = "match" // match + OpCIDR Op = "cidr" // cidr + OpHas Op = "has" // has + OpNotHas Op = "not_has" // not has + OpPrefix Op = "prefix" // prefix + OpRe Op = "re" // regex + OpIn Op = "in" // in + OpNotIn Op = "not_in" // not in + OpNotCIDR Op = "not_cidr" // not cidr + OpExist Op = "exist" // exist + OpNotExist Op = "not_exist" // not exist + OpGeoEq Op = "geo_eq" // geo equal + OpGeoNotEq Op = "geo_not_eq" // geo not equal +) + +type Pattern struct { + K Key `json:"k"` + Op Op `json:"op"` + V []string `json:"v"` + SubK string `json:"sub_k"` +} diff --git a/mcp_server/internal/config/config.go b/mcp_server/internal/config/config.go new file mode 100644 index 0000000..30a3dd7 --- /dev/null +++ b/mcp_server/internal/config/config.go @@ -0,0 +1,118 @@ +package config + +import ( + "os" + "strconv" + + "github.com/chaitin/SafeLine/mcp_server/pkg/errors" + "gopkg.in/yaml.v3" +) + +// Config Global configuration structure +type Config struct { + Server *ServerConfig `yaml:"server"` + Logger *LoggerConfig `yaml:"logger"` + API *APIConfig `yaml:"api"` +} + +// APIConfig API configuration +type APIConfig struct { + // API base URL + BaseURL string `yaml:"base_url"` + // API token + Token string `yaml:"token"` + // API timeout + Timeout int `yaml:"timeout"` + // API debug mode + Debug bool `yaml:"debug"` + // API insecure skip verify + InsecureSkipVerify bool `yaml:"insecure_skip_verify"` +} + +// ServerConfig Server configuration +type ServerConfig struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + Port int `yaml:"port"` + Host string `yaml:"host"` + Secret string `yaml:"secret"` +} + +// LoggerConfig Logger configuration +type LoggerConfig struct { + Level string `yaml:"level"` + FilePath string `yaml:"file_path"` + Console bool `yaml:"console"` + Caller bool `yaml:"caller"` + Development bool `yaml:"development"` +} + +var config *Config + +// getEnvString Get string value from environment variable, return default value if not exists +func getEnvString(key, defaultValue string) string { + if value, exists := os.LookupEnv(key); exists { + return value + } + return defaultValue +} + +// getEnvInt Get integer value from environment variable, return default value if not exists or cannot be parsed +func getEnvInt(key string, defaultValue int) int { + if value, exists := os.LookupEnv(key); exists { + if intValue, err := strconv.Atoi(value); err == nil { + return intValue + } + } + return defaultValue +} + +// Load Load configuration file +func Load(path string) error { + data, err := os.ReadFile(path) + if err != nil { + return errors.Wrap(err, "read config file failed") + } + + config = &Config{} + if err := yaml.Unmarshal(data, config); err != nil { + return errors.Wrap(err, "unmarshal config failed") + } + + // Override configuration from environment variables + if config.Server != nil { + config.Server.Host = getEnvString("LISTEN_ADDRESS", config.Server.Host) + config.Server.Port = getEnvInt("LISTEN_PORT", config.Server.Port) + } + + if config.API != nil { + config.API.BaseURL = getEnvString("SAFELINE_ADDRESS", config.API.BaseURL) + config.API.Token = getEnvString("SAFELINE_API_TOKEN", config.API.Token) + } + + return nil +} + +// GetServer Get server configuration +func GetServer() *ServerConfig { + if config == nil { + return nil + } + return config.Server +} + +// GetLogger Get logger configuration +func GetLogger() *LoggerConfig { + if config == nil { + return nil + } + return config.Logger +} + +// GetAPI Get API configuration +func GetAPI() *APIConfig { + if config == nil { + return nil + } + return config.API +} diff --git a/mcp_server/internal/tools/analyze/get_atttack_events.go b/mcp_server/internal/tools/analyze/get_atttack_events.go new file mode 100644 index 0000000..a0ec1b3 --- /dev/null +++ b/mcp_server/internal/tools/analyze/get_atttack_events.go @@ -0,0 +1,45 @@ +package analyze + +import ( + "context" + + "github.com/chaitin/SafeLine/mcp_server/internal/api/analyze" + "github.com/chaitin/SafeLine/mcp_server/pkg/logger" +) + +type GetAttackEventsParams struct { + IP string `json:"ip" desc:"ip" required:"false"` + Page int `json:"page" desc:"page" required:"false" default:"1"` + PageSize int `json:"page_size" desc:"page size" required:"false" default:"10"` + Start int64 `json:"start" desc:"start unix timestamp in milliseconds" required:"false"` + End int64 `json:"end" desc:"end unix timestamp in milliseconds" required:"false"` +} + +type GetAttackEvents struct{} + +func (t *GetAttackEvents) Name() string { + return "get_attack_events" +} + +func (t *GetAttackEvents) Description() string { + return "get attack events" +} + +func (t *GetAttackEvents) Validate(params GetAttackEventsParams) error { + return nil +} + +func (t *GetAttackEvents) Execute(ctx context.Context, params GetAttackEventsParams) (analyze.GetEventListResponse, error) { + resp, err := analyze.GetEventList(ctx, &analyze.GetEventListRequest{ + IP: params.IP, + PageSize: params.PageSize, + Page: params.Page, + Start: params.Start, + End: params.End, + }) + if err != nil { + return analyze.GetEventListResponse{}, err + } + logger.With("total", resp.Total).Info("get attack events") + return *resp, nil +} diff --git a/mcp_server/internal/tools/app/create_application.go b/mcp_server/internal/tools/app/create_application.go new file mode 100644 index 0000000..96809ab --- /dev/null +++ b/mcp_server/internal/tools/app/create_application.go @@ -0,0 +1,41 @@ +package app + +import ( + "context" + + "github.com/chaitin/SafeLine/mcp_server/internal/api/app" + "github.com/chaitin/SafeLine/mcp_server/pkg/logger" +) + +type CreateApp struct{} + +type CreateAppParams struct { + ServerNames []string `json:"server_names" desc:"domain list" required:"true"` + Ports []string `json:"ports" desc:"port list" required:"true"` + Upstreams []string `json:"upstreams" desc:"upstream list" required:"true"` +} + +func (t *CreateApp) Name() string { + return "create_http_application" +} + +func (t *CreateApp) Description() string { + return "create a new website or app" +} + +func (t *CreateApp) Validate(params CreateAppParams) error { + return nil +} + +func (t *CreateApp) Execute(ctx context.Context, params CreateAppParams) (int64, error) { + id, err := app.CreateApp(ctx, &app.CreateAppRequest{ + ServerNames: params.ServerNames, + Ports: params.Ports, + Upstreams: params.Upstreams, + }) + if err != nil { + return 0, err + } + logger.Info("create app success", logger.Int64("id", id)) + return id, nil +} diff --git a/mcp_server/internal/tools/example.go b/mcp_server/internal/tools/example.go new file mode 100644 index 0000000..d482bc2 --- /dev/null +++ b/mcp_server/internal/tools/example.go @@ -0,0 +1,46 @@ +package tools + +import ( + "context" + + "github.com/chaitin/SafeLine/mcp_server/pkg/errors" + "github.com/chaitin/SafeLine/mcp_server/pkg/logger" +) + +type CalculateSum struct{} + +func (t *CalculateSum) Name() string { + return "calculate_sum" +} + +func (t *CalculateSum) Description() string { + return "Add two numbers together" +} + +type MyToolInput struct { + A int `json:"a" desc:"number a" required:"true"` + B int `json:"b" desc:"number b" required:"true"` +} + +type MyToolOutput struct { + C int `json:"c"` +} + +func (t *CalculateSum) Validate(params MyToolInput) error { + return nil +} + +func (t *CalculateSum) Execute(ctx context.Context, params MyToolInput) (MyToolOutput, error) { + logger.With("a", params.A). + With("b", params.B). + Debug("Executing calculation") + + result := MyToolOutput{ + C: params.A + params.B, + } + + logger.With("result", result.C). + Debug("Calculation completed") + + return result, errors.New("test error") +} diff --git a/mcp_server/internal/tools/init.go b/mcp_server/internal/tools/init.go new file mode 100644 index 0000000..03f850c --- /dev/null +++ b/mcp_server/internal/tools/init.go @@ -0,0 +1,19 @@ +package tools + +import ( + "github.com/chaitin/SafeLine/mcp_server/internal/tools/analyze" + "github.com/chaitin/SafeLine/mcp_server/internal/tools/app" + "github.com/chaitin/SafeLine/mcp_server/internal/tools/rule" +) + +func init() { + // app + AppendTool(&app.CreateApp{}) + + // rule + AppendTool(&rule.CreateBlacklistRule{}) + AppendTool(&rule.CreateWhitelistRule{}) + + // analyze + AppendTool(&analyze.GetAttackEvents{}) +} diff --git a/mcp_server/internal/tools/rule/create_blacklist_rule.go b/mcp_server/internal/tools/rule/create_blacklist_rule.go new file mode 100644 index 0000000..3e7b81b --- /dev/null +++ b/mcp_server/internal/tools/rule/create_blacklist_rule.go @@ -0,0 +1,66 @@ +package rule + +import ( + "context" + + "github.com/chaitin/SafeLine/mcp_server/internal/api" + "github.com/chaitin/SafeLine/mcp_server/internal/api/rule" + "github.com/chaitin/SafeLine/mcp_server/pkg/logger" +) + +type CreateBlacklistRule struct{} + +type CreateBlacklistRuleParams struct { + Name string `json:"name" desc:"name" required:"true"` + IP []string `json:"ip" desc:"ip" required:"false"` + URINoQuery []string `json:"uri_no_query" desc:"uri_no_query" required:"false"` +} + +func (t *CreateBlacklistRule) Name() string { + return "create_blacklist_rule" +} + +func (t *CreateBlacklistRule) Description() string { + return "create a new blacklist rule" +} + +func (t *CreateBlacklistRule) Validate(params CreateBlacklistRuleParams) error { + return nil +} + +func (t *CreateBlacklistRule) Execute(ctx context.Context, params CreateBlacklistRuleParams) (int64, error) { + var pattern [][]api.Pattern + if len(params.IP) > 0 { + pattern = append(pattern, []api.Pattern{ + { + K: api.KeySrcIP, + Op: api.OpEq, + V: params.IP, + SubK: "", + }, + }) + } + if len(params.URINoQuery) > 0 { + pattern = append(pattern, []api.Pattern{ + { + K: api.KeyURINoQuery, + Op: api.OpEq, + V: params.URINoQuery, + SubK: "", + }, + }) + } + + id, err := rule.CreateRule(ctx, &rule.CreateRuleRequest{ + Name: params.Name, + IP: params.IP, + IsEnabled: true, + Action: int(api.PolicyRuleActionDeny), + Pattern: pattern, + }) + if err != nil { + return 0, err + } + logger.With("id", id).Info("create blacklist rule success") + return id, nil +} diff --git a/mcp_server/internal/tools/rule/create_whitelist_rule.go b/mcp_server/internal/tools/rule/create_whitelist_rule.go new file mode 100644 index 0000000..4b83f84 --- /dev/null +++ b/mcp_server/internal/tools/rule/create_whitelist_rule.go @@ -0,0 +1,66 @@ +package rule + +import ( + "context" + + "github.com/chaitin/SafeLine/mcp_server/internal/api" + "github.com/chaitin/SafeLine/mcp_server/internal/api/rule" + "github.com/chaitin/SafeLine/mcp_server/pkg/logger" +) + +type CreateWhitelistRule struct{} + +type CreateWhitelistRuleParams struct { + Name string `json:"name" desc:"name" required:"true"` + IP []string `json:"ip" desc:"ip" required:"false"` + URINoQuery []string `json:"uri_no_query" desc:"uri_no_query" required:"false"` +} + +func (t *CreateWhitelistRule) Name() string { + return "create_whitelist_rule" +} + +func (t *CreateWhitelistRule) Description() string { + return "create a new whitelist rule" +} + +func (t *CreateWhitelistRule) Validate(params CreateWhitelistRuleParams) error { + return nil +} + +func (t *CreateWhitelistRule) Execute(ctx context.Context, params CreateWhitelistRuleParams) (int64, error) { + var pattern [][]api.Pattern + if len(params.IP) > 0 { + pattern = append(pattern, []api.Pattern{ + { + K: api.KeySrcIP, + Op: api.OpEq, + V: params.IP, + SubK: "", + }, + }) + } + if len(params.URINoQuery) > 0 { + pattern = append(pattern, []api.Pattern{ + { + K: api.KeyURINoQuery, + Op: api.OpEq, + V: params.URINoQuery, + SubK: "", + }, + }) + } + + id, err := rule.CreateRule(ctx, &rule.CreateRuleRequest{ + Name: params.Name, + IP: params.IP, + IsEnabled: true, + Action: int(api.PolicyRuleActionAllow), + Pattern: pattern, + }) + if err != nil { + return 0, err + } + logger.With("id", id).Info("create whitelist rule success") + return id, nil +} diff --git a/mcp_server/internal/tools/tool.go b/mcp_server/internal/tools/tool.go new file mode 100644 index 0000000..6a57c42 --- /dev/null +++ b/mcp_server/internal/tools/tool.go @@ -0,0 +1,41 @@ +package tools + +import ( + "github.com/chaitin/SafeLine/mcp_server/pkg/logger" + "github.com/chaitin/SafeLine/mcp_server/pkg/mcp" +) + +// By deferring the concretization of generic types to the Register method, +// we avoid type inference issues. + +// Each Tool is wrapped in a toolWrapper that knows its concrete type, +// allowing correct passing of generic parameters during registration. +type ToolWrapper interface { + Register(s *mcp.MCPServer) error +} + +var ( + tools = []ToolWrapper{} +) + +func AppendTool[T any, R any](tool ...mcp.Tool[T, R]) { + for _, t := range tool { + tools = append(tools, &toolWrapper[T, R]{tool: t}) + } +} + +func Tools() []ToolWrapper { + return tools +} + +type toolWrapper[T any, R any] struct { + tool mcp.Tool[T, R] +} + +func (w *toolWrapper[T, R]) Register(s *mcp.MCPServer) error { + logger.Info("Registering tool", + logger.String("name", w.tool.Name()), + logger.String("description", w.tool.Description()), + ) + return mcp.RegisterTool(s, w.tool) +} diff --git a/mcp_server/main.go b/mcp_server/main.go new file mode 100644 index 0000000..9a8dcb0 --- /dev/null +++ b/mcp_server/main.go @@ -0,0 +1,62 @@ +package main + +import ( + "flag" + "fmt" + + "github.com/chaitin/SafeLine/mcp_server/internal/api" + "github.com/chaitin/SafeLine/mcp_server/internal/config" + "github.com/chaitin/SafeLine/mcp_server/internal/tools" + "github.com/chaitin/SafeLine/mcp_server/pkg/logger" + "github.com/chaitin/SafeLine/mcp_server/pkg/mcp" +) + +func main() { + configPath := flag.String("config", "config.yaml", "path to config file") + flag.Parse() + + if err := config.Load(*configPath); err != nil { + panic(fmt.Errorf("failed to load config: %v", err)) + } + + logConfig := config.GetLogger() + if err := logger.Init(&logger.Config{ + Level: logConfig.Level, + FilePath: logConfig.FilePath, + Console: logConfig.Console, + Caller: logConfig.Caller, + Development: logConfig.Development, + }); err != nil { + panic(fmt.Errorf("failed to init logger: %v", err)) + } + + logger.With("base_url", config.GetAPI().BaseURL).Info("Initializing API service...") + if err := api.Init(config.GetAPI()); err != nil { + panic(fmt.Errorf("failed to init API service: %v", err)) + } + + logger.Info("Starting MCP Server...") + serverConfig := config.GetServer() + s := mcp.NewMCPServer( + serverConfig.Name, + serverConfig.Version, + serverConfig.Secret, + ) + + logger.Info("Registering tools...") + for _, tool := range tools.Tools() { + if err := tool.Register(s); err != nil { + logger.With("error", err). + Error("Failed to register tool") + panic(err) + } + } + + addr := fmt.Sprintf("%s:%d", serverConfig.Host, serverConfig.Port) + logger.With("addr", addr).Info("Starting server") + if err := s.Start(addr); err != nil { + logger.With("error", err). + Error("Server failed to start") + panic(err) + } +} diff --git a/mcp_server/pkg/config/config.go b/mcp_server/pkg/config/config.go new file mode 100644 index 0000000..a4d93f1 --- /dev/null +++ b/mcp_server/pkg/config/config.go @@ -0,0 +1,72 @@ +package config + +import ( + "os" + + "gopkg.in/yaml.v3" +) + +// Config Global configuration structure +type Config struct { + Server ServerConfig `yaml:"server"` + Logger LoggerConfig `yaml:"logger"` +} + +// ServerConfig Server configuration +type ServerConfig struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + Port int `yaml:"port"` + Host string `yaml:"host"` + Secret string `yaml:"secret"` +} + +// LoggerConfig Logger configuration +type LoggerConfig struct { + Level string `yaml:"level"` + FilePath string `yaml:"file_path"` + Console bool `yaml:"console"` + Caller bool `yaml:"caller"` + Development bool `yaml:"development"` +} + +var ( + globalConfig *Config +) + +// Load Load configuration from file +func Load(filename string) error { + data, err := os.ReadFile(filename) + if err != nil { + return err + } + + config := &Config{} + if err := yaml.Unmarshal(data, config); err != nil { + return err + } + + globalConfig = config + return nil +} + +// Get Get global configuration +func Get() *Config { + return globalConfig +} + +// GetServer Get server configuration +func GetServer() ServerConfig { + if globalConfig == nil { + return ServerConfig{} + } + return globalConfig.Server +} + +// GetLogger Get logger configuration +func GetLogger() LoggerConfig { + if globalConfig == nil { + return LoggerConfig{} + } + return globalConfig.Logger +} diff --git a/mcp_server/pkg/errors/errors.go b/mcp_server/pkg/errors/errors.go new file mode 100644 index 0000000..ee6ebae --- /dev/null +++ b/mcp_server/pkg/errors/errors.go @@ -0,0 +1,136 @@ +package errors + +import ( + "errors" + "fmt" + "runtime" + "strings" + + "github.com/chaitin/SafeLine/mcp_server/pkg/logger" +) + +var ( + // Common errors + ErrInternal = New("internal error") + ErrInvalidParam = New("invalid parameter") + ErrNotFound = New("resource not found") + ErrUnauthorized = New("unauthorized") + ErrForbidden = New("forbidden") + ErrTimeout = New("timeout") +) + +// Error Custom error structure +type Error struct { + err error + stack []string + msg string + location string +} + +// Error Implement error interface +func (e *Error) Error() string { + if e.msg != "" { + return fmt.Sprintf("%s: %v (at %s)", e.msg, e.err, e.location) + } + return fmt.Sprintf("%v (at %s)", e.err, e.location) +} + +// Unwrap Return original error +func (e *Error) Unwrap() error { + if e.err == nil { + return nil + } + if wrapped, ok := e.err.(*Error); ok { + return wrapped.Unwrap() + } + return e.err +} + +// Stack Return error stack +func (e *Error) Stack() []string { + return e.stack +} + +// Location Return error location +func (e *Error) Location() string { + return e.location +} + +// getCallerLocation Get caller location +func getCallerLocation(skip int) string { + _, file, line, ok := runtime.Caller(skip) + if !ok { + return "unknown" + } + return fmt.Sprintf("%s:%d", file, line) +} + +// WrapL Wrap error and print log +func WrapL(err error, msg string) error { + if err == nil { + return nil + } + + // Get stack trace information + var stack []string + for i := 1; i < 32; i++ { + pc, file, line, ok := runtime.Caller(i) + if !ok { + break + } + fn := runtime.FuncForPC(pc) + if fn == nil { + break + } + name := fn.Name() + if strings.Contains(name, "runtime.") { + break + } + stack = append(stack, fmt.Sprintf("%s:%d", file, line)) + } + + wrappedErr := &Error{ + err: err, + stack: stack, + msg: msg, + location: getCallerLocation(2), + } + + // Print error information and stack using logger + logger.With("error", err). + With("location", wrappedErr.location). + With("stack", strings.Join(stack, "\n")). + Error(msg) + + return wrappedErr +} + +// Is Check error type +func Is(err, target error) bool { + return errors.Is(err, target) +} + +// As Type assertion +func As(err error, target interface{}) bool { + return errors.As(err, target) +} + +// Wrap Wrap error without printing log +func Wrap(err error, msg string) error { + if err == nil { + return nil + } + return &Error{ + err: err, + msg: msg, + location: getCallerLocation(2), + } +} + +// New Create new error +func New(text string) error { + return &Error{ + err: errors.New(text), + location: getCallerLocation(2), + } +} diff --git a/mcp_server/pkg/logger/field.go b/mcp_server/pkg/logger/field.go new file mode 100644 index 0000000..fa185a9 --- /dev/null +++ b/mcp_server/pkg/logger/field.go @@ -0,0 +1,49 @@ +package logger + +import ( + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// Field 日志字段 +type Field = zapcore.Field + +// String 创建字符串字段 +func String(key string, val string) Field { + return zap.String(key, val) +} + +// Int 创建整数字段 +func Int(key string, val int) Field { + return zap.Int(key, val) +} + +// Int64 创建 int64 字段 +func Int64(key string, val int64) Field { + return zap.Int64(key, val) +} + +// Float64 创建浮点数字段 +func Float64(key string, val float64) Field { + return zap.Float64(key, val) +} + +// Bool 创建布尔字段 +func Bool(key string, val bool) Field { + return zap.Bool(key, val) +} + +// Err 创建错误字段 +func Err(err error) Field { + return zap.Error(err) +} + +// Any 创建任意类型字段 +func Any(key string, val interface{}) Field { + return zap.Any(key, val) +} + +// Duration 创建时间段字段 +func Duration(key string, val float64) Field { + return zap.Float64(key, val) +} diff --git a/mcp_server/pkg/logger/logger.go b/mcp_server/pkg/logger/logger.go new file mode 100644 index 0000000..e52ad25 --- /dev/null +++ b/mcp_server/pkg/logger/logger.go @@ -0,0 +1,205 @@ +package logger + +import ( + "os" + "path/filepath" + "sync" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// Logger 封装 zap.Logger +type Logger struct { + zl *zap.Logger +} + +var ( + defaultLogger *Logger + once sync.Once +) + +// Config 日志配置 +type Config struct { + // 日志级别 + Level string `json:"level" yaml:"level"` + // 日志文件路径 + FilePath string `json:"file_path" yaml:"file_path"` + // 是否输出到控制台 + Console bool `json:"console" yaml:"console"` + // 是否记录调用者信息 + Caller bool `json:"caller" yaml:"caller"` + // 是否使用开发模式（更详细的日志） + Development bool `json:"development" yaml:"development"` +} + +// 默认配置 +var defaultConfig = Config{ + Level: "info", + FilePath: "logs/mcp.log", + Console: true, + Caller: true, + Development: false, +} + +// Init 初始化日志系统 +func Init(cfg *Config) error { + var err error + once.Do(func() { + if cfg == nil { + cfg = &defaultConfig + } + + // 确保日志目录存在 + if cfg.FilePath != "" { + dir := filepath.Dir(cfg.FilePath) + if err = os.MkdirAll(dir, 0755); err != nil { + return + } + } + + // 配置编码器 + encoderConfig := zapcore.EncoderConfig{ + TimeKey: "time", + LevelKey: "level", + NameKey: "logger", + CallerKey: "caller", + MessageKey: "msg", + StacktraceKey: "stacktrace", + LineEnding: zapcore.DefaultLineEnding, + EncodeLevel: zapcore.LowercaseLevelEncoder, + EncodeTime: func(t time.Time, enc zapcore.PrimitiveArrayEncoder) { + enc.AppendString(t.Format("2006-01-02 15:04:05.000")) + }, + EncodeDuration: zapcore.SecondsDurationEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + } + + // 设置日志级别 + var level zapcore.Level + if err = level.UnmarshalText([]byte(cfg.Level)); err != nil { + level = zapcore.InfoLevel + } + + // 创建Core + var cores []zapcore.Core + + // 文件输出 + if cfg.FilePath != "" { + fileWriter, err := os.OpenFile(cfg.FilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return + } + cores = append(cores, zapcore.NewCore( + zapcore.NewJSONEncoder(encoderConfig), + zapcore.AddSync(fileWriter), + level, + )) + } + + // 控制台输出 + if cfg.Console { + cores = append(cores, zapcore.NewCore( + zapcore.NewConsoleEncoder(encoderConfig), + zapcore.AddSync(os.Stdout), + level, + )) + } + + // 创建Logger + core := zapcore.NewTee(cores...) + zl := zap.New(core) + + // 是否记录调用者信息 + if cfg.Caller { + zl = zl.WithOptions(zap.AddCaller()) + } + + // 是否使用开发模式 + if cfg.Development { + zl = zl.WithOptions(zap.Development()) + } + + defaultLogger = &Logger{zl: zl} + }) + + return err +} + +// GetLogger 获取日志实例 +func GetLogger() *Logger { + if defaultLogger == nil { + Init(nil) + } + return defaultLogger +} + +// With 创建带有字段的新Logger +func (l *Logger) With(key string, value interface{}) *Logger { + return &Logger{zl: l.zl.With(Any(key, value))} +} + +// Debug level +func (l *Logger) Debug(msg string, fields ...Field) { + l.zl.Debug(msg, fields...) +} + +// Info level +func (l *Logger) Info(msg string, fields ...Field) { + l.zl.Info(msg, fields...) +} + +// Warn level +func (l *Logger) Warn(msg string, fields ...Field) { + l.zl.Warn(msg, fields...) +} + +// Error level +func (l *Logger) Error(msg string, fields ...Field) { + l.zl.Error(msg, fields...) +} + +// Fatal level +func (l *Logger) Fatal(msg string, fields ...Field) { + l.zl.Fatal(msg, fields...) +} + +// 全局函数 + +// With 创建带有字段的新Logger +func With(key string, value interface{}) *Logger { + l := GetLogger() + // 为链式调用创建新的logger实例，并添加caller skip + return &Logger{zl: l.zl.WithOptions(zap.AddCallerSkip(1)).With(Any(key, value))} +} + +// Debug level +func Debug(msg string, fields ...Field) { + l := GetLogger() + l.zl.WithOptions(zap.AddCallerSkip(1)).Debug(msg, fields...) +} + +// Info level +func Info(msg string, fields ...Field) { + l := GetLogger() + l.zl.WithOptions(zap.AddCallerSkip(1)).Info(msg, fields...) +} + +// Warn level +func Warn(msg string, fields ...Field) { + l := GetLogger() + l.zl.WithOptions(zap.AddCallerSkip(1)).Warn(msg, fields...) +} + +// Error level +func Error(msg string, fields ...Field) { + l := GetLogger() + l.zl.WithOptions(zap.AddCallerSkip(1)).Error(msg, fields...) +} + +// Fatal level +func Fatal(msg string, fields ...Field) { + l := GetLogger() + l.zl.WithOptions(zap.AddCallerSkip(1)).Fatal(msg, fields...) +} diff --git a/mcp_server/pkg/mcp/mcp.go b/mcp_server/pkg/mcp/mcp.go new file mode 100644 index 0000000..55be4e4 --- /dev/null +++ b/mcp_server/pkg/mcp/mcp.go @@ -0,0 +1,148 @@ +package mcp + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + + "github.com/chaitin/SafeLine/mcp_server/pkg/errors" + "github.com/chaitin/SafeLine/mcp_server/pkg/logger" + "github.com/mark3labs/mcp-go/mcp" + "github.com/mark3labs/mcp-go/server" + "github.com/mcuadros/go-defaults" +) + +type Tool[T any, R any] interface { + Name() string + + Description() string + + Execute(ctx context.Context, params T) (R, error) + + Validate(params T) error +} + +type SSEServer struct { + sse *server.SSEServer + secret string +} + +func (s *SSEServer) Start(addr string) error { + srv := &http.Server{ + Addr: addr, + Handler: s, + } + return srv.ListenAndServe() +} + +func (s *SSEServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if s.secret == "" { + s.sse.ServeHTTP(w, r) + return + } + messagePath := s.sse.CompleteMessagePath() + if messagePath != "" && r.URL.Path == messagePath { + secret := r.Header.Get("Secret") + if secret != s.secret { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + } + s.sse.ServeHTTP(w, r) +} + +type MCPServer struct { + server *server.MCPServer + sse *SSEServer +} + +func NewMCPServer(name, version string, secret string) *MCPServer { + s := server.NewMCPServer( + name, + version, + server.WithLogging(), + ) + return &MCPServer{ + server: s, + sse: &SSEServer{sse: server.NewSSEServer(s), secret: secret}, + } +} + +func (s *MCPServer) Start(addr string) error { + return s.sse.Start(addr) +} + +func handleToolCall[T any, R any](ctx context.Context, request mcp.CallToolRequest, tool Tool[T, R]) (result *mcp.CallToolResult, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic recovered: %v", r) + } + }() + + var raw []byte + raw, err = json.Marshal(request.Params.Arguments) + if err != nil { + return nil, errors.Wrap(err, "marshal arguments failed") + } + var params T + defaults.SetDefaults(¶ms) + if err = json.Unmarshal(raw, ¶ms); err != nil { + return nil, errors.Wrap(err, "unmarshal parameters failed") + } + + if err = tool.Validate(params); err != nil { + return nil, err + } + + var execResult R + execResult, err = tool.Execute(ctx, params) + if err != nil { + return nil, err + } + + v := any(execResult) + switch v := v.(type) { + case string: + return mcp.NewToolResultText(v), nil + case []byte: + return mcp.NewToolResultText(string(v)), nil + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: + return mcp.NewToolResultText(json.Number(fmt.Sprint(v)).String()), nil + case bool: + return mcp.NewToolResultText(strconv.FormatBool(v)), nil + default: + bytes, err := json.Marshal(v) + if err != nil { + return nil, errors.New("invalid result type") + } + return mcp.NewToolResultText(string(bytes)), nil + } +} + +func RegisterTool[T any, R any](s *MCPServer, tool Tool[T, R]) error { + var v T + opts, err := SchemaToOptions(v) + if err != nil { + return err + } + opts = append(opts, mcp.WithDescription(tool.Description())) + t := mcp.NewTool(tool.Name(), + opts..., + ) + + s.server.AddTool(t, func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + result, err := handleToolCall(ctx, request, tool) + if err != nil { + logger.With("error", err).Error("handle tool call failed") + if wrapped, ok := err.(*errors.Error); ok { + return nil, wrapped.Unwrap() + } + return nil, err + } + return result, nil + }) + + return nil +} diff --git a/mcp_server/pkg/mcp/schema.go b/mcp_server/pkg/mcp/schema.go new file mode 100644 index 0000000..7461563 --- /dev/null +++ b/mcp_server/pkg/mcp/schema.go @@ -0,0 +1,169 @@ +package mcp + +import ( + "encoding/json" + "reflect" + "strconv" + "strings" + + "github.com/mark3labs/mcp-go/mcp" +) + +// SchemaToOptions Convert struct to MCP ToolOption list +func SchemaToOptions(schema any) ([]mcp.ToolOption, error) { + t := reflect.TypeOf(schema) + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + + var options []mcp.ToolOption + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + jsonTag := field.Tag.Get("json") + if jsonTag == "" { + continue + } + + desc := field.Tag.Get("desc") + required := field.Tag.Get("required") == "true" + enumTag := field.Tag.Get("enum") + defaultTag := field.Tag.Get("default") + minTag := field.Tag.Get("min") + maxTag := field.Tag.Get("max") + opts := []mcp.PropertyOption{} + + if desc != "" { + opts = append(opts, mcp.Description(desc)) + } + if required { + opts = append(opts, mcp.Required()) + } + if enumTag != "" && field.Type.Kind() == reflect.String { + enumValues := strings.Split(enumTag, ",") + opts = append(opts, mcp.Enum(enumValues...)) + } + + switch field.Type.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64: + if defaultTag != "" { + if defaultValue, err := strconv.Atoi(defaultTag); err == nil { + opts = append(opts, mcp.DefaultNumber(float64(defaultValue))) + } + } + if minTag != "" { + if minValue, err := strconv.Atoi(minTag); err == nil { + opts = append(opts, mcp.Min(float64(minValue))) + } + } + if maxTag != "" { + if maxValue, err := strconv.Atoi(maxTag); err == nil { + opts = append(opts, mcp.Max(float64(maxValue))) + } + } + options = append(options, mcp.WithNumber(jsonTag, opts...)) + case reflect.Bool: + if defaultTag != "" { + if defaultValue, err := strconv.ParseBool(defaultTag); err == nil { + opts = append(opts, mcp.DefaultBool(defaultValue)) + } + } + options = append(options, mcp.WithBoolean(jsonTag, opts...)) + case reflect.String: + if defaultTag != "" { + opts = append(opts, mcp.DefaultString(defaultTag)) + } + options = append(options, mcp.WithString(jsonTag, opts...)) + case reflect.Struct: + subSchema := reflect.New(field.Type).Interface() + subOptions, err := SchemaToOptions(subSchema) + if err != nil { + return nil, err + } + + // Create a temporary Tool to get JSON Schema of sub-struct + tempTool := mcp.NewTool("temp", subOptions...) + tempJSON, _ := tempTool.MarshalJSON() + var tempMap map[string]any + if err := json.Unmarshal(tempJSON, &tempMap); err != nil { + continue + } + + // Extract properties from temporary Tool + if inputSchema, ok := tempMap["inputSchema"].(map[string]any); ok { + if properties, ok := inputSchema["properties"].(map[string]any); ok { + // Check if there are required fields + if required, ok := inputSchema["required"].([]any); ok { + // Add required field information to corresponding properties + for _, req := range required { + if reqStr, ok := req.(string); ok { + if prop, ok := properties[reqStr].(map[string]any); ok { + prop["required"] = true + } + } + } + } + opts = append(opts, mcp.Properties(properties)) + } + } + options = append(options, mcp.WithObject(jsonTag, opts...)) + + case reflect.Slice: + elemType := field.Type.Elem() + var items map[string]any + + switch elemType.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64: + items = map[string]any{ + "type": "number", + } + case reflect.Bool: + items = map[string]any{ + "type": "boolean", + } + case reflect.String: + items = map[string]any{ + "type": "string", + } + case reflect.Struct: + subSchema := reflect.New(elemType).Interface() + subOptions, err := SchemaToOptions(subSchema) + if err != nil { + return nil, err + } + + // Create a temporary Tool to get JSON Schema of sub-struct + tempTool := mcp.NewTool("temp", subOptions...) + tempJSON, _ := tempTool.MarshalJSON() + var tempMap map[string]any + if err := json.Unmarshal(tempJSON, &tempMap); err != nil { + continue + } + + // Extract properties from temporary Tool + if inputSchema, ok := tempMap["inputSchema"].(map[string]any); ok { + if properties, ok := inputSchema["properties"].(map[string]any); ok { + // Check if there are required fields + if required, ok := inputSchema["required"].([]any); ok { + // Add required field information to corresponding properties + for _, req := range required { + if reqStr, ok := req.(string); ok { + if prop, ok := properties[reqStr].(map[string]any); ok { + prop["required"] = true + } + } + } + } + items = map[string]any{ + "type": "object", + "properties": properties, + } + } + } + } + opts = append(opts, mcp.Items(items)) + options = append(options, mcp.WithArray(jsonTag, opts...)) + } + } + + return options, nil +} diff --git a/mcp_server/pkg/mcp/schema_test.go b/mcp_server/pkg/mcp/schema_test.go new file mode 100644 index 0000000..6025f74 --- /dev/null +++ b/mcp_server/pkg/mcp/schema_test.go @@ -0,0 +1,227 @@ +package mcp + +import ( + "reflect" + "testing" + + "github.com/mark3labs/mcp-go/mcp" +) + +func TestSchemaToOptions(t *testing.T) { + tests := []struct { + name string + args any + want mcp.Tool + }{ + { + name: "test number", + args: struct { + A int `json:"a" desc:"number a" required:"true"` + }{}, + want: mcp.NewTool("test number", + mcp.WithNumber("a", mcp.Required(), mcp.Description("number a")), + ), + }, + { + name: "test number int64", + args: struct { + A int64 `json:"a" desc:"number a" required:"true"` + }{}, + want: mcp.NewTool("test number int64", + mcp.WithNumber("a", mcp.Required(), mcp.Description("number a")), + ), + }, + { + name: "test number float64", + args: struct { + A float64 `json:"a" desc:"number a" required:"true"` + }{}, + want: mcp.NewTool("test number float64", + mcp.WithNumber("a", mcp.Required(), mcp.Description("number a")), + ), + }, + { + name: "test number default", + args: struct { + A int `json:"a" desc:"number a" required:"true" default:"10"` + }{}, + want: mcp.NewTool("test number default", + mcp.WithNumber("a", mcp.Required(), mcp.Description("number a"), mcp.DefaultNumber(10)), + ), + }, + { + name: "test number min max", + args: struct { + A int `json:"a" desc:"number a" required:"true" min:"10" max:"20"` + }{}, + want: mcp.NewTool("test number min max", + mcp.WithNumber("a", mcp.Required(), mcp.Description("number a"), mcp.Min(10), mcp.Max(20)), + ), + }, + { + name: "test number optional", + args: struct { + A int `json:"a" desc:"number a"` + }{}, + want: mcp.NewTool("test number optional", + mcp.WithNumber("a", mcp.Description("number a")), + ), + }, + { + name: "test boolean", + args: struct { + A bool `json:"a" desc:"boolean a" required:"true"` + }{}, + want: mcp.NewTool("test boolean", + mcp.WithBoolean("a", mcp.Required(), mcp.Description("boolean a")), + ), + }, + { + name: "test string", + args: struct { + A string `json:"a" desc:"string a" required:"true"` + }{}, + want: mcp.NewTool("test string", + mcp.WithString("a", mcp.Required(), mcp.Description("string a")), + ), + }, + { + name: "test string default", + args: struct { + A string `json:"a" desc:"string a" required:"true" default:"hello"` + }{}, + want: mcp.NewTool("test string default", + mcp.WithString("a", mcp.Required(), mcp.Description("string a"), mcp.DefaultString("hello")), + ), + }, + { + name: "test string enum", + args: struct { + A string `json:"a" desc:"string a" required:"true" enum:"1,2,3"` + }{}, + want: mcp.NewTool("test string enum", + mcp.WithString("a", mcp.Required(), mcp.Description("string a"), mcp.Enum("1", "2", "3")), + ), + }, + { + name: "test object", + args: struct { + A struct { + B int `json:"b" desc:"number b" required:"true"` + } `json:"a" desc:"object a" required:"true"` + }{}, + want: mcp.NewTool("test object", + mcp.WithObject("a", mcp.Required(), mcp.Description("object a"), + mcp.Properties(map[string]any{ + "b": map[string]any{ + "type": "number", + "description": "number b", + "required": true, + }, + }), + ), + ), + }, + { + name: "test object optional", + args: struct { + A struct { + B int `json:"b" desc:"number b" required:"true"` + C int `json:"c" desc:"number c"` + } `json:"a" desc:"object a" required:"true"` + }{}, + want: mcp.NewTool("test object optional", + mcp.WithObject("a", mcp.Required(), mcp.Description("object a"), + mcp.Properties(map[string]any{ + "b": map[string]any{ + "type": "number", + "description": "number b", + "required": true, + }, + "c": map[string]any{ + "type": "number", + "description": "number c", + }, + }), + ), + ), + }, + { + name: "test nested object", + args: struct { + A struct { + B struct { + C int `json:"c" desc:"number c" required:"true"` + } `json:"b" desc:"object b" required:"true"` + } `json:"a" desc:"object a" required:"true"` + }{}, + want: mcp.NewTool("test nested object", + mcp.WithObject("a", mcp.Required(), mcp.Description("object a"), + mcp.Properties(map[string]any{ + "b": map[string]any{ + "type": "object", + "description": "object b", + "required": true, + "properties": map[string]any{ + "c": map[string]any{ + "type": "number", + "description": "number c", + "required": true, + }, + }, + }, + }), + ), + ), + }, + { + name: "test array", + args: struct { + A []int `json:"a" desc:"array a" required:"true"` + }{}, + want: mcp.NewTool("test array", + mcp.WithArray("a", mcp.Required(), mcp.Description("array a"), + mcp.Items(map[string]any{ + "type": "number", + }), + ), + ), + }, + { + name: "test array of object", + args: struct { + A []struct { + B int `json:"b" desc:"number b" required:"true"` + } `json:"a" desc:"array of object a" required:"true"` + }{}, + want: mcp.NewTool("test array of object", + mcp.WithArray("a", mcp.Required(), mcp.Description("array of object a"), + mcp.Items(map[string]any{ + "type": "object", + "properties": map[string]any{ + "b": map[string]any{ + "type": "number", + "description": "number b", + "required": true, + }, + }, + }), + ), + ), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := SchemaToOptions(tt.args) + if err != nil { + t.Errorf("SchemaToOptions() error = %v", err) + return + } + s1, _ := mcp.NewTool(tt.name, got...).MarshalJSON() + s2, _ := tt.want.MarshalJSON() + if !reflect.DeepEqual(s1, s2) { + t.Errorf("\n got %v\n want %v", string(s1), string(s2)) + } + }) + } +} diff --git a/scripts/manage.py b/scripts/manage.py new file mode 100644 index 0000000..831a6dc --- /dev/null +++ b/scripts/manage.py @@ -0,0 +1,1567 @@ +#!/usr/bin/env python3 +import argparse +import base64 +import json +import shutil +import ssl +import sys +import datetime +import platform +import os +from urllib.request import urlopen, HTTPError +import re +import subprocess +import string +import time +import random +import socket + + +texts = { + 'hello1': { + 'en': 'SafeLine is a self-hosted WAF(Web Application Firewall) to protect your web apps from attacks and exploits.', + 'zh': 'SafeLine，中文名 "雷池"，是一款简单好用, 效果突出的 Web 应用防火墙(WAF)，可以保护 Web 服务不受黑客攻击。' + }, + 'hello2': { + 'en': 'A web application firewall helps protect web apps by filtering and monitoring HTTP traffic between a web application and the Internet. It typically protects web apps from attacks such as SQL injection, XSS, code injection, os command injection, CRLF injection, ldap injection, xpath injection, RCE, XXE, SSRF, path traversal, backdoor, bruteforce, http-flood, bot abused, among others.', + 'zh': '雷池通过过滤和监控 Web 应用与互 联网之间的 HTTP 流量来保护 Web 服务。可以保护 Web 服务免受 SQL 注入、XSS 、 代码注入、命令注入、CRLF 注入、ldap 注入、xpath 注入、RCE、XXE、SSRF、路径遍历、后门、暴力破解、CC、爬虫 等攻击。' + }, + 'talking-group': { + 'en': '\n' + 'https://discord.gg/SVnZGzHFvn\n' + '\n' + 'Join discord group for more informations of SafeLine by above address', + 'zh': '\n' + '▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄\n' + '█ ▄▄▄▄▄ █▀ █▀▀██▀▄▀▀▄▀▄▀▄██ ▄▄▄▄▄ █\n' + '█ █ █ █▀ ▄ █▀▄▄▀▀ ▄█▄ ▀█ █ █ █\n' + '█ █▄▄▄█ █▀█ █▄█▄▀▀▄▀▄ ▀▀▄▄█ █▄▄▄█ █\n' + '█▄▄▄▄▄▄▄█▄█▄█ █▄▀ █ ▀▄▀ █▄█▄▄▄▄▄▄▄█\n' + '█▄ ▄▄ █▄▄ ▄█▄▄▄▄▀▄▀▀▄██ ▄▄▀▄█▄▀ ▀█\n' + '█▄ ▄▀▄ ▄▀▄ ▀ ▄█▀ ▀▄ █▀▀ ▀█▀▄██▄▀▄██\n' + '██ ▀▄█ ▄ ▄▄▀▄▀▀█▄▀▄▄▀▄▀▄ ▄ ▀▄▄▄█▀▀█\n' + '█ █▀▄▀ ▄▀▄▄▀█▀ ▄▄ █▄█▀▀▄▀▀█▄█▄█▀▄██\n' + '█ █ ▀ ▄▀▀ ██▄█▄▄▄▄▄▀▄▀▀▀▄▄▀█▄▀█ ▀█\n' + '█ █ ▀▄ ▄██▀▀ ▄█▀ ▀███▄ ▀▄▀▄▄ ▄▀▄██\n' + '█▀▄▄█ ▄▀▄▀ ▄▀▀▀▄▀▄▀ ▄▀▄ ▄▀ ▄▀█ ▀█\n' + '█ █ █ █▄▀ █▄█▀ ▄▄███▀▀▀▄█▀▄ ▀ ▀▄██\n' + '█▄███▄█▄▄▀▄ █▄█▄▄▄▄▀▀▄█▀▀ ▄▄▄ ▀█ █\n' + '█ ▄▄▄▄▄ █▄▀█ ▄█▀▄ █▀█▄ ▀ █▄█ ▀▄▀█\n' + '█ █ █ █ █▄▀▀▀▄▄▄▀▀▀▀▀▀ ▄▄ ▀█ █\n' + '█ █▄▄▄█ █ ▀█▀ ▄▄▄▄ ▀█ ▀▀▄▀ ▀▀ ▀███\n' + '█▄▄▄▄▄▄▄█▄▄██▄█▄▄█▄██▄██▄▄█▄▄█▄█▄██\n' + '\n' + '微信扫描上方二维码加入雷池项目讨论组' + }, + 'input-target-path': { + 'en': 'Input the path to install %s', + 'zh': '请输入%s 的安装目录' + }, + 'input-mgt-port': { + 'en': 'Input the %s mgt port', + 'zh': '请输入%s 的管理端口' + }, + 'python-version-too-low': { + 'en': 'The Python version is too low, Python 3.5 above is required', + 'zh': 'Python 版本过低, 脚本无法运行, 需要 Python 3.5 以上' + }, + 'not-a-tty': { + 'en': 'Stdin is not a standard TTY', + 'zh': '运行脚本的方式不对, STDIN 不是标准的 TTY' + }, + 'not-root': { + 'en': 'Requires root privileges to run', + 'zh': '需要 root 权限才能运行' + }, + 'not-linux': { + 'en': '%s does not support %s OS yet', + 'zh': '%s 暂时不支持 %s 操作系统' + }, + 'unsupported-arch': { + 'en': '%s does not support %s processor yet', + 'zh': '%s 暂时不支持 %s 处理器' + }, + 'prepare-to-install': { + 'en': 'Will be going to installing %s for you.', + 'zh': '即将为您安装%s' + }, + 'choice-action': { + 'en': 'Choice what do you want to do', + 'zh': '选择你要执行的动作' + }, + 'default-value': { + 'en': 'Keep blank default to', + 'zh': '留空则为默认值' + }, + 'ssse3-not-support': { + 'en': 'SSSE3 instruction set not enabled in your CPU', + 'zh': '当前 CPU 未启用 SSSE3 指令集' + }, + 'precheck-failed': { + 'en': 'The environment does not meet the installation conditions of %s', + 'zh': '当前环境不符合%s 的安装条件' + }, + 'precheck-passed': { + 'en': 'Installation environment check passed', + 'zh': '检查安装环境已完成' + }, + 'insufficient-memory': { + 'en': 'Remaining memory is less than 1 GB', + 'zh': '剩余内存不足 1 GB' + }, + 'docker-not-installed': { + 'en': 'Running %s requires Docker, but Docker is not installed', + 'zh': '运行%s 依赖 Docker, 但是 Docker 没安装' + }, + 'docker-compose-not-installed': { + 'en': 'Running %s requires Docker Compose, but Docker Compose is not installed', + 'zh': '运行%s 依赖 Docker Compose, 但是 Docker Compose 没安装' + }, + 'docker-version-too-low': { + 'en': 'Docker version is too low, it does not match %s', + 'zh': 'Docker 版本太低, 不满足%s 的安装需求' + }, + 'if-install-docker': { + 'en': 'Do you want the latest version of Docker to be automatically installed for you?', + 'zh': '是否需要为你自动安装 Docker 的最新版本' + }, + 'if-restart-docker': { + 'en': 'Do you want to restart %s Docker container', + 'zh': '是否需要重启%s 的容器' + }, + 'if-update-docker': { + 'en': 'Do you want to update your Docker version?', + 'zh': '是否需要为你自动更新 Docker 版本' + }, + 'install-docker-failed': { + 'en': 'Failed to install Docker. Please try to install Docker manually before installing %s', + 'zh': '安装 Docker 失败, 请尝试手动安装 Docker 后再来安装%s' + }, + 'install-docker': { + 'en': 'Docker is being installed for you. It will take a few minutes. Please wait patiently.', + 'zh': '正在为你安装 Docker, 需要几分钟时间, 请耐心等待' + }, + 'get-space-failed': { + 'en': 'Unable to query disk capacity of "%s"', + 'zh': '无法查询 "%s" 的磁盘容量' + }, + 'remain-disk-capacity': { + 'en': 'Disk capacity of "%s" has %s avaiable', + 'zh': '"%s" 路径有 %s 的空间可用' + }, + 'insufficient-disk-capacity': { + 'en': 'Insufficient disk capacity of "%s", at least 5 GB is required to install %s', + 'zh': '"%s" 的磁盘容量不足，安装%s 至少需要 5 GB' + }, + 'pg-pass-contains-invalid-char': { + 'en': 'The POSTGRES_PASSWORD variable contains special characters. Please choose repair to reset password', + 'zh': 'POSTGRES_PASSWORD 变量包含特殊字符, 请选择修复重置密码' + }, + 'invalid-path': { + 'en': '"%s" is not a valid absolute path', + 'zh': '"%s" 不是合法的绝对路径' + }, + 'path-exists': { + 'en': '"%s" already exists, please select a new directory', + 'zh': '路径 "%s" 已存在，请选择一个全新的目录' + }, + 'fail-to-parse-route': { + 'en': 'Unable to parse /proc/net/route file', + 'zh': '无法解析 /proc/net/route 文件' + }, + 'fail-to-download-compose': { + 'en': 'Failed to download docker compose script', + 'zh': '下载 docker compose 脚本失败' + }, + 'fail-to-create-dir': { + 'en': 'Unable to create the "%s" directory', + 'zh': '无法创建 "%s" 目录' + }, + 'docker-pull': { + 'en': 'Pulling Docker image', + 'zh': '正在拉取 Docker 镜像' + }, + 'try-another-image-source': { + 'en': 'Try another image source', + 'zh': '尝试使用其他镜像源' + }, + 'image-clean': { + 'en': 'Cleaning Docker image', + "zh": '正在清理 Docker 镜像' + }, + 'update-config': { + 'en': 'Updating .env configuration files', + 'zh': '正在更新 .env 配置文件' + }, + 'download-compose': { + 'en': 'Downloading the docker-compose.yaml file', + 'zh': '正在下载 docker-compose.yaml 文件' + }, + 'fail-to-pull-image': { + 'en': 'Failed to pull Docker image', + 'zh': '拉取 Docker 镜像失败' + }, + 'docker-up': { + 'en': 'Starting Docker containers', + 'zh': '正在启动 Docker 容器' + }, + 'fail-to-up': { + 'en': 'Failed to start Docker containers', + 'zh': '启动 Docker 容器失败' + }, + 'fail-to-down': { + 'en': 'Failed to stop Docker containers', + 'zh': '停止 Docker 容器失败' + }, + 'install-finish': { + 'en': '%s installation completed', + 'zh': '%s 安装完成' + }, + 'upgrade-finish': { + 'en': '%s upgrade completed', + 'zh': '%s 升级完成' + }, + 'go-to-panel': { + 'en': '%s management panel: https://%s:%s/', + 'zh': '%s 管理面板: https://%s:%s/' + } + ,'install': { + 'en': 'INSTALL', + 'zh': '安装' + }, + 'repair': { + 'en': 'REPAIR', + 'zh': '修复' + }, + 'uninstall': { + 'en': 'UNINSTALL', + 'zh': '卸载' + }, + 'upgrade': { + 'en': 'UPGRADE', + 'zh': '升级' + }, + 'backup': { + 'en': 'BACKUP', + 'zh': '备份' + }, + 'yes': { + 'en': 'Yes', + 'zh': '是' + }, + 'no': { + 'en': 'No', + 'zh': '否' + }, + 'fail-to-get-installed-dir': { + 'en': 'Failed to get installed dir', + 'zh': '未能找到安装目录', + }, + 'fail-to-connect-image-source': { + 'en': 'Failed to connect image source', + 'zh': '无法连接到任何镜像源' + }, + 'fail-to-connect-docker-source': { + 'en': 'Failed to connect docker source', + 'zh': '无法连接到任何 docker 源' + }, + 'fail-to-download-docker-installation': { + 'en': 'Failed to download docker installation', + "zh": '下载 docker 安装脚本失败' + }, + 'docker-source': { + 'en': 'Docker source', + "zh": 'docker 源' + }, + 'reset-admin': { + 'en': 'Setup admin', + "zh": '设置 admin' + }, + 'docker-version': { + 'en': 'Checking docker version', + 'zh': '检查 docker 版本' + }, + 'docker-compose-version': { + 'en': 'Checking docker compose version', + 'zh': '检查 docker compose 版本' + }, + 'keyboard-interrupt': { + 'en': 'Installation cancelled', + "zh": '取消安装' + }, + 'docker-up-iptables-failed': { + 'en': 'Iptables policy error, try to restart docker', + 'zh': 'iptables 规则错误，尝试重启 docker' + }, + 'install-channel': { + 'en': 'Installing: %s', + 'zh': '安装通道：%s' + }, + 'preview-release': { + 'en': 'Preview', + 'zh': '预览版' + }, + 'lts-release': { + 'en': 'LTS', + 'zh': 'LTS 版' + }, + 'fail-to-docker-down': { + 'en': 'Failed to stop container', + 'zh': '停止 docker 容器失败' + }, + 'fail-to-remove-dir': { + 'en': 'Failed to remove %s installation directory', + 'zh': '删除 %s 安装目录失败' + }, + 'uninstall-finish': { + 'en': '%s uninstall completed', + 'zh': '%s 卸载完成' + }, + 'docker-down': { + 'en': 'Stopping %s container', + 'zh': '正在停止%s 容器' + }, + 'reset-tengine': { + 'en': 'RESET TENGINE CONFIG', + 'zh': '重置 tengine 配置', + }, + 'reset-postgres': { + 'en': 'RESET DATABASE PASSWORD', + 'zh': '重置数据库密码' + }, + 'fail-to-find-nginx': { + 'en': 'Failed to find tengine config path', + 'zh': '未找到 tengine 配置目录' + }, + 'nginx-backup-dir': { + 'en': 'Tengine config backup directory', + 'zh': 'tengine 配置备份目录' + }, + 'fail-to-backup-nginx': { + 'en': 'Failed to backup tengine config', + 'zh': '备份 tengine 目录失败' + }, + 'docker-restart': { + 'en': 'Restart docker container', + 'zh': '重启 docker 容器' + }, + 'docker-exec': { + 'en': 'Executing docker command', + 'zh': '执行 docker 命令' + }, + 'fail-to-recover-static': { + 'en': 'Failed to recover tengine static config', + 'zh': '恢复 tengine 静态站点资源失败' + }, + 'fail-to-find-env': { + 'en': 'Failed to find .env file', + 'zh': '未找到 .env 文件' + }, + 'fail-to-find-postgres-password': { + 'en': 'Failed to find postgres password', + 'zh': '未找到数据库密码' + }, + 'fail-to-reset-postgres-password': { + 'en': 'Failed to reset postgres password', + 'zh': '重置数据库密码失败' + }, + 'reset-postgres-password-finish': { + 'en': 'Reset postgres password completed', + 'zh': '重置数据库密码完成' + }, + 'reset-tengine-finish': { + 'en': 'Reset tengine finish completed', + 'zh': '重置 tengine 配置完成' + }, + 'if-remove-waf': { + 'en': 'Do you want to uninstall %s, this operation will delete all data in the directory: %s', + 'zh': '是否确认卸载%s，该操作会删除目录下所有数据：%s' + }, + 'restart-docker-finish': { + 'en': 'Restart %s docker container completed', + 'zh': '重启%s 容器完成' + }, + 'restart': { + 'en': 'RESTART', + 'zh': '重启' + }, + 'wait-mgt-health': { + 'en': 'Wait for mgt healthy', + 'zh': '等待 mgt 启动' + }, + 'if-remove-ipv6-scope': { + 'en': '/etc/resolv.conf have ipv6 nameserver with scope. Do you want to remove these nameserver', + 'zh': '/etc/resolv.conf 文件中存在 ipv6 地址包含区域 ID，是否删除包含区域 ID 的 ipv6 nameserver' + }, + 'input-target-version': { + 'en': 'Input the %s version', + 'zh': '请输入 %s 的版本' + }, + 'version-format-error': { + 'en': 'version %s format error', + 'zh': '版本 %s 格式错误' + }, + 'can-not-downgrade': { + 'en': '%s can not downgrade %s to %s', + 'zh': '%s 不支持从 %s 降级到 %s' + }, + 'get-version': { + 'en': 'Getting %s latest version', + 'zh': '正在获取%s 最新版本' + }, + 'get-version-from-mgt': { + 'en': 'try to get version from mgt', + 'zh': '尝试从 mgt 获取安装版本' + }, + 'skip-version-compare': { + 'en': 'skip %s version compare', + 'zh': '跳过%s 版本匹配' + }, + 'fail-to-get-version-from-mgt': { + 'en': 'failed to get install version from mgt', + 'zh': '从 mgt 获取安装版本失败' + }, + 'target-version': { + 'en': 'target version: %s', + 'zh': '目标版本：%s' + }, + 'fail-to-parse-assets': { + 'en': 'failed to parse assets', + 'zh': '解析补丁包失败' + }, + 'snap-docker-should-use-home-path': { + 'en': 'Docker installed via snap can only be configured to set the installation directory under the user\'s home directory(%s)', + 'zh': 'snap 安装的 docker 只能设置安装目录在用户的主目录(%s)下' + }, + 'fail-to-get-docker-path': { + 'en': 'find docker binary path failed', + 'zh': '获取 docker 二进制目录失败' + } +} + + +BOLD = 1 +DIM = 1 +BLINK = 5 +REVERSE = 7 +RED = 31 +GREEN = 32 +YELLOW = 33 +BLUE = 34 +CYAN = 36 + +INSTALL = False +DOMAIN = 'waf-ce.chaitin.cn' +REQUEST_CTX = None +LANG = 'zh' +PRODUCT = '' +DEBUG = False +SELF = True + +def parse_assets(args): + global PRODUCT, SELF + + if args.patch == '': + return True + assets = '' + with open(args.patch, 'r') as f: + for line in f.readlines(): + assets += line + + split_assets = assets.strip().split('.') + if len(split_assets) != 3: + return False + + try: + assets_info = json.loads(base64.b64decode(split_assets[1].replace('_','/').replace('-','+') + '=' * ((4 - len(split_assets[1]) % 4) % 4))) + if args.en: + PRODUCT = assets_info['fullname_en'] + else: + PRODUCT = assets_info['fullname'] + + if PRODUCT == '': + return False + + if assets_info['self'] is None: + return False + + if not assets_info['self']: + SELF = False + except Exception as e: + log.debug('parse assets failed: ' + str(e)) + return False + + return True + +def init_global_config(): + global REQUEST_CTX, LANG, DOMAIN, PRODUCT, DEBUG + + REQUEST_CTX = ssl.create_default_context() + REQUEST_CTX.check_hostname = False + REQUEST_CTX.verify_mode = ssl.CERT_NONE + + parser = argparse.ArgumentParser( + prog='installer-management', + description='installer-management', + allow_abbrev=False + ) + parser.add_argument('--debug', action='store_true', help='install with debug log') + parser.add_argument("--lts", action='store_true', help='install lts version') + parser.add_argument('--image-clean', action='store_true', help='clean image when upgrade done') + parser.add_argument('--en', action='store_true', help='install international version') + parser.add_argument('--patch', default='', type=str, help='patch path') + args = parser.parse_args() + if args.en: + LANG = 'en' + DOMAIN = 'waf.chaitin.com' + PRODUCT = 'SafeLine WAF' + else: + PRODUCT = '雷池 WAF' + + if args.debug: + DEBUG = True + + if args.patch != '' and not os.path.exists(args.patch): + log.fatal('assets %s not exists' % args.patch) + + if not parse_assets(args): + log.fatal(text('fail-to-parse-assets')) + return args + +class log(): + @staticmethod + def _log(c, l, s): + t = datetime.datetime.now().strftime('%H:%M:%S') + print('\r\033[0;%dm[%-5s %s]: %s\033[0m' % (c, l, t, s)) + + @staticmethod + def debug(s): + if DEBUG: + log._log(DIM, 'DEBUG', s) + + @staticmethod + def info(s): + log._log(CYAN, 'INFO', s) + + @staticmethod + def warning(s): + log._log(YELLOW, 'WARN', s) + + @staticmethod + def error(s): + log._log(RED, 'ERROR', s) + + @staticmethod + def fatal(s): + log._log(RED, 'ERROR', s) + sys.exit(1) + +def text(label, var=()): + t = texts.get(label, { + 'en': 'Unknown "%s" (%s)' % (label, var), + 'zh': '未知变量 "%s" (%s)' % (label, var) + }) + return t[LANG if LANG in t else 'en'] % var + +def color(t, attrs=[], end=True): + t = '\x1B[%sm%s' % (';'.join([str(i) for i in attrs]), t) + if end: + t = t + '\x1B[m' + return t + +GLOBAL_ARGS = init_global_config() + +def banner(): + t = r''' + ______ ___ _____ _ ____ ____ _ ________ +.' ____ \ .' ..] |_ _| (_) |_ _| |_ _| / \ |_ __ | +| (___ \_| ,--. _| |_ .---. | | __ _ .--. .---. \ \ /\ / / / _ \ | |_ \_| + _.____`. `'_\ : '-| |-' / /__\\ | | _ [ | [ `.-. | / /__\\ \ \/ \/ / / ___ \ | _| +| \____) | // | |, | | | \__., _| |__/ | | | | | | | | \__., \ /\ / _/ / \ \_ _| |_ + \______.' \'-;__/ [___] '.__.' |________| [___] [___||__] '.__.' \/ \/ |____| |____| |_____| + +'''.strip('\n') + print(color(t + '\n', [GREEN, BLINK])) + +def get_url(url): + try: + response = urlopen(url, timeout=10, context=REQUEST_CTX) + content = response.read() + return content.decode('utf-8') + except Exception as e: + log.error('get url %s failed: %s' % (url, str(e))) + +def ui_read(question, default): + while True: + if default is None: + sys.stdout.write('%s: ' % ( + color(question, [GREEN]), + )) + else: + sys.stdout.write('%s %s: ' % ( + color(question, [GREEN]), + color('(%s %s)' % (text('default-value'), default), [YELLOW]), + )) + r = input().strip() + if len(r) == 0: + if default is None or len(default) == 0: + continue + r = default + return r + +def ui_choice(question, options): + while True: + s_options = '[ %s ]' % ' '.join(['%s.%s' % option for option in options]) + s_choices = '(%s)' % '/'.join([option[0] for option in options]) + sys.stdout.write('%s %s %s: ' % (color(question, [GREEN]), color(s_options, [YELLOW]), color(s_choices, [YELLOW]))) + r = input().strip() + if r in [i[0] for i in options]: + return r + +def humen_size(x): + if x >= 1024 * 1024 * 1024 * 1024: + return '%.02f TB' % (x / 1024 / 1024 / 1024 / 1024) + elif x >= 1024 * 1024 * 1024: + return '%.02f GB' % (x / 1024 / 1024 / 1024) + elif x >= 1024 * 1024: + return '%.02f MB' % (x / 1024 / 1024) + elif x >= 1024: + return '%.02f KB' % (x / 1024) + else: + return '%d Bytes' + +def rand_subnet(): + routes = [] + try: + with open('/proc/net/route', 'r') as f: + next(f) + for line in f: + parts = line.split() + if len(parts) < 8: + continue + destination = int(parts[1], 0x10) + if destination == 0: + continue + mask = int(parts[7], 0x10) + routes.append((destination, mask)) + except Exception as e: + log.error(text('fail-to-parse-route') + ' ' + str(e)) + for i in range(256): + t = 192 + t += 168 << 8 + t += i << 16 + for route in routes: + if t & route[1] == route[0]: + break + else: + return '%d.%d.%d' % (t & 0xFF, (t >> 8) & 0xFF, (t >> 16) & 0xFF) + return '172.22.222' + +def free_space(path): + while not os.path.exists(path) and path != '/': + path = os.path.dirname(path) + try: + st = os.statvfs(path) + free_bytes = st.f_bavail * st.f_frsize + return free_bytes + except Exception as e: + log.error(text('get-space-failed', path) + ' ' + str(e)) + return None + +def free_memory(): + t = filter(lambda x: 'MemAvailable' in x, open('/proc/meminfo', 'r').readlines()) + return int(next(t).split()[1]) * 1024 + +def exec_command(*args,shell=False): + try: + proc = subprocess.run(args, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,shell=shell) + subprocess_output(proc.stdout.strip()) + return proc.returncode, proc.stdout, proc.stderr + except Exception as e: + return -1, '', str(e) + +def exec_command_with_loading(*args, cwd=None, env=None): + try: + with subprocess.Popen(args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, env=env, cwd=cwd) as proc: + if not DEBUG: + loading = ["⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"] + iloading = 0 + while proc.poll() is None: + sys.stderr.write('\r' + loading[iloading]) + sys.stderr.flush() + iloading = (iloading + 1) % len(loading) + time.sleep(0.1) + sys.stderr.write('\r') + sys.stderr.flush() + else: + for line in iter(proc.stdout.readline, b''): + if line.strip() != '': + log.debug(" -->> "+line.strip()) + if proc.poll() is not None and line == '': + break + return proc.returncode, proc.stdout.read(), proc.stderr.read() + except Exception as e: + return -1, '', str(e) + +def subprocess_output(stdout): + if stdout != '': + log.debug(" -->> "+stdout) + else: + log.debug(" -->> subprocess empty output") + +def start_docker(): + return exec_command('systemctl enable docker && systemctl daemon-reload && systemctl restart docker',shell=True) + +def check_port(port): + if not INSTALL: + return True + + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(('0.0.0.0', int(port))) + return True + except Exception as e: + log.debug("try listen mgt port failed: "+str(e)) + return False + +def install_docker(): + log.info(text('install-docker')) + + log.debug("downloading get-docker.sh") + if not save_file_from_url('https://'+DOMAIN+'/release/latest/get-docker.sh','get-docker.sh'): + raise Exception(text('fail-to-download-docker-installation')) + + source = docker_source() + if source == '': + raise Exception(text('fail-to-connect-docker-source')) + + log.debug(text('docker-source')+': '+source) + env = { + "DOWNLOAD_URL": source, + "https_proxy": os.environ.get('https_proxy',''), + } + log.debug("installing docker") + r = exec_command_with_loading('bash get-docker.sh',env=env) + if r[0] == 0: + p = start_docker() + subprocess_output(p[1].strip()) + if p[0] != 0: + log.error("start docker failed: "+p[2].strip()) + return p[0] == 0 + else: + log.error("install docker error: "+r[2].strip()) + return False + +compose_command = '' + +def precheck_docker_compose(): + log.info(text("docker-compose-version")) + global compose_command + + while True: + version_output = '' + proc = exec_command('docker', 'compose', 'version') + if proc[0] == 0: + help_proc = exec_command('docker', 'compose', 'up', '--help') + if help_proc[0] == 0 and '--detach' in help_proc[1]: + compose_command = 'docker compose -f docker-compose.yaml' + version_output = proc[1].strip() + else: + log.debug('docker compose can not find detach argument') + else: + compose_proc = exec_command('docker-compose', 'version') + if compose_proc[0] == 0: + help_proc = exec_command('docker-compose', 'up', '--help') + if help_proc[0] == 0 and '--detach' in help_proc[1]: + compose_command = 'docker-compose -f docker-compose.yaml' + version_output = compose_proc[1].strip() + else: + log.debug('docker-compose can not find detach argument') + else: + log.warning(text('docker-compose-not-installed', PRODUCT)) + + if version_output != '': + t = re.findall(r'^Docker Compose version v?(\d+)\.', version_output) + if len(t) == 0: + log.warning(text('docker-compose-not-installed', PRODUCT)) + elif int(t[0]) < 2: + log.warning(text('docker-version-too-low', PRODUCT)) + else: + return True + + action = ui_choice(text('if-update-docker'), [ + ('y', text('yes')), + ('n', text('no')), + ]) + if action.lower() == 'n': + return False + elif action.lower() == 'y': + if not install_docker(): + log.warning(text('install-docker-failed', PRODUCT)) + return False + +def precheck_dns_scope(): + resolve_file = '/etc/resolv.conf' + if not os.path.exists(resolve_file): + return True + + have_scope = False + raw_lines = [] + with open(resolve_file, 'r') as f: + for line in f.readlines(): + strip_line = line.strip() + if not strip_line.startswith('nameserver') or '%' not in strip_line.lstrip('nameserver').strip(): + raw_lines.append(line) + continue + + have_scope = True + + if not have_scope: + return True + + action = ui_choice(text('if-remove-ipv6-scope'),[ + ('y', text('yes')), + ('n', text('no')), + ]) + if action.lower() != 'y': + return False + + shutil.copyfile(resolve_file, resolve_file+'.bak') + with open(resolve_file, 'w') as f: + for line in raw_lines: + f.write(line) + + return True + +def precheck(): + if platform.machine() in ('x86_64', 'AMD64') and 'ssse3' not in open('/proc/cpuinfo', 'r').read().lower(): + log.warning(text('ssse3-not-support')) + return False + + if free_memory() < 1024 * 1024 * 1024: + log.warning(text('insufficient-memory')) + return False + + log.info(text("docker-version")) + while True: + proc = exec_command('docker', '--version') + if proc[0] == 0: + t = re.findall(r'^Docker version (\d+)\.', proc[1]) + if len(t) == 0: + log.warning(text('docker-not-installed', PRODUCT)) + elif int(t[0]) < 20: + log.warning(text('docker-version-too-low', PRODUCT)) + else: + break + else: + log.warning(text('docker-not-installed', PRODUCT)) + + action = ui_choice(text('if-install-docker'), [ + ('y', text('yes')), + ('n', text('no')), + ]) + if action.lower() == 'n': + return False + elif action.lower() == 'y': + if not install_docker(): + log.warning(text('install-docker-failed', PRODUCT)) + return False + + if not precheck_docker_compose(): + return False + + if not precheck_dns_scope(): + return False + + return True + +def docker_pull(cwd): + log.info(text('docker-pull')) + try: + subprocess.check_call(compose_command+' pull', cwd=cwd, shell=True) + return True + except Exception as e: + log.warning("docker pull error: "+str(e)) + return False + +def docker_restart(container): + log.info(text('docker-restart')+": "+container) + try: + subprocess.check_call('docker restart '+container, shell=True) + return True + except Exception as e: + log.error("docker restart error: "+str(e)) + return False + +def docker_exec(container, command): + log.info(text('docker-exec')+": ("+container+") "+command) + try: + subprocess.check_call('docker exec '+container+' '+command, shell=True) + return True + except Exception as e: + log.error("docker exec error: "+str(e)) + return False + +def image_clean(): + log.info(text('image-clean')) + proc = exec_command('docker image prune -f --filter="label=maintainer=SafeLine-CE"', shell=True) + if proc[0] != 0: + log.warning("remove docker image failed: "+proc[2]) + +def docker_up(cwd): + log.info(text('docker-up')) + while True: + p = exec_command_with_loading(compose_command+' up -d --remove-orphans', cwd=cwd) + if p[0] == 0: + return True + if 'iptables failed' in p[2]: + log.warning("docker up error: "+p[2]) + while True: + action = ui_choice(text('docker-up-iptables-failed'),[ + ('y', text('yes')), + ('n', text('no')), + ]) + if action.lower() == 'y': + start_docker() + break + elif action.lower() == 'n': + return False + else: + log.error("docker up error: "+p[2]) + return False + +def docker_down(cwd): + log.info(text('docker-down', PRODUCT)) + try: + subprocess.check_call(compose_command+' down', cwd=cwd, shell=True) + return True + except Exception: + return False + +def get_url_time(url): + now = datetime.datetime.now() + try: + urlopen(url, timeout=10, context=REQUEST_CTX) + except HTTPError as e: + log.debug("get url "+url+" status: "+str(e.status)) + if e.status > 499: + return 999999 + except Exception as e: + log.debug("get url "+url+" failed: "+str(e)) + return 999999 + return (datetime.datetime.now() - now).microseconds / 1000 + +def get_avg_delay(url): + log.debug("test url avg delay: "+url) + total_delay = 0 + for i in range(3): + total_delay += get_url_time(url) + + avg_delay = total_delay / 3 + log.debug("url "+url+" avg delay: "+str(avg_delay)) + return avg_delay + +pull_failed_prefix = [] + +def image_source(): + source = { + 'https://registry-1.docker.io': 'chaitin', + "https://swr.cn-east-3.myhuaweicloud.com": 'swr.cn-east-3.myhuaweicloud.com/chaitin-safeline' + } + + min_delay = -1 + image_prefix = '' + + for url, prefix in source.items(): + if prefix in pull_failed_prefix: + continue + delay = get_avg_delay(url) + if delay > 0 and (min_delay < 0 or delay < min_delay): + min_delay = delay + image_prefix = prefix + + log.debug("use image_prefix: "+image_prefix) + return image_prefix + +def docker_source(): + sources = [ + "https://mirrors.aliyun.com/docker-ce/", + "https://mirrors.tencent.com/docker-ce/", + "https://download.docker.com" + ] + + min_delay = -1 + source = '' + for v in sources: + delay = get_avg_delay(v) + if delay > 0 and (min_delay < 0 or delay < min_delay): + min_delay = delay + source = v + return source + +def read_config(path,config): + with open(path, 'r') as f: + for line in f.readlines(): + if line.strip() == '': + continue + try: + s = line.index('=') + if s > 0: + k = line[:s].strip() + v = line[s + 1:].strip() + config[k] = v + except ValueError: + continue + +def write_config(path,config): + with open(path, 'w') as f: + for k in config: + f.write('%s=%s\n' % (k, config[k])) + +def generate_config(path): + log.info(text('update-config')) + config = { + 'SAFELINE_DIR': path, + 'POSTGRES_PASSWORD': '', + 'MGT_PORT': '', + 'RELEASE': '', + 'CHANNEL': '', + 'REGION': '', + 'IMAGE_PREFIX': '', + 'IMAGE_TAG': '', + 'SUBNET_PREFIX': '', + 'ARCH_SUFFIX': '' + } + + env_path = os.path.join(path,'.env') + if os.path.exists(env_path): + read_config(env_path,config) + + if config['ARCH_SUFFIX'] == '': + if platform.machine() == 'aarch64': + config['ARCH_SUFFIX'] = '-arm' + + if config['POSTGRES_PASSWORD'] == '': + config['POSTGRES_PASSWORD'] = ''.join([random.choice(string.ascii_letters + string.digits) for i in range(20)]) + + if config['SUBNET_PREFIX'] == '': + config['SUBNET_PREFIX'] = rand_subnet() + + if config['RELEASE'] == '' and GLOBAL_ARGS.lts: + config['RELEASE'] = '-lts' + config['CHANNEL'] = '-lts' + + if config['RELEASE'] == '-lts': + GLOBAL_ARGS.lts = True + + default_try = False + if config['MGT_PORT'] == '9443': + default_try = True + while not config['MGT_PORT'].isnumeric() or int(config['MGT_PORT']) >= 65536 or int(config['MGT_PORT']) <= 0 or not check_port(config['MGT_PORT']): + if not default_try: + config['MGT_PORT'] = '9443' + default_try = True + else: + config['MGT_PORT'] = ui_read(text('input-mgt-port', PRODUCT),None) + + if config['REGION'] == '' and GLOBAL_ARGS.en: + config['REGION'] = '-g' + + if not config['POSTGRES_PASSWORD'].isalnum(): + log.info(text('pg-pass-contains-invalid-char')) + raise Exception(text('pg-pass-contains-invalid-char')) + + if config['IMAGE_PREFIX'] == '' or config['IMAGE_PREFIX'] in pull_failed_prefix: + config['IMAGE_PREFIX'] = image_source() + if config['IMAGE_PREFIX'] == '': + raise Exception(text('fail-to-connect-image-source')) + + config['IMAGE_TAG'] = get_version(config['IMAGE_TAG']) + log.info(text('target-version', config['IMAGE_TAG'])) + + write_config(env_path, config) + return config + +def show_address(mgt_port): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + local_ip = s.getsockname()[0] + log.info(text('go-to-panel', (PRODUCT, local_ip, mgt_port))) + log.info(text('go-to-panel', (PRODUCT, '0.0.0.0', mgt_port))) + +def init_mgt(): + while True: + p = exec_command('docker', 'inspect','--format=\'{{.State.Health.Status}}\'', 'safeline-mgt') + if p[0] == 0 and p[1].strip().replace("'",'') == 'healthy': + break + elif p[0] != 0: + log.debug("get safeline-mgt status error: "+str(p[2])) + log.info(text('wait-mgt-health')) + time.sleep(5) + + log.info(text('reset-admin')) + proc = exec_command('docker exec safeline-mgt /app/mgt-cli reset-admin --once',shell=True) + if proc[0] != 0: + log.warning(proc[2]) + elif proc[1].strip() != '': + log.info('\n'+proc[1].strip()) + +def check_install_path(safeline_path): + if not safeline_path.startswith('/'): + return False + proc = exec_command('which', 'docker') + if proc[0] != 0: + log.debug('get docker path failed: '+proc[2]) + raise Exception(text('fail-to-get-docker-path')) + if not proc[1].startswith('/snap'): + return True + home_path = os.path.expanduser('~') + if not safeline_path.startswith(home_path): + log.warning(text('snap-docker-should-use-home-path', home_path)) + return False + return True + +def install(): + global INSTALL + INSTALL = True + log.info(text('prepare-to-install', PRODUCT)) + + if not precheck(): + log.error(text('precheck-failed', PRODUCT)) + return + log.info(text('precheck-passed')) + + default_path = '/data/safeline' + if not SELF: + default_path = '/data/waf' + + while True: + safeline_path = ui_read(text('input-target-path', PRODUCT), default_path) + if not check_install_path(safeline_path): + log.warning(text('invalid-path', safeline_path)) + continue + if os.path.exists(safeline_path): + log.warning(text('path-exists', safeline_path)) + continue + if free_space(safeline_path) < 5 * 1024 * 1024 * 1024: + log.warning(text('insufficient-disk-capacity', (safeline_path, PRODUCT))) + continue + break + + try: + os.makedirs(safeline_path) + except Exception as e: + log.error(text('fail-to-create-dir', safeline_path) + ' ' + str(e)) + return + + mgt_path = os.path.join(safeline_path,'resources','mgt') + try: + os.makedirs(mgt_path, exist_ok=True) + except Exception as e: + log.error(text('fail-to-create-dir', mgt_path) + ' ' + str(e)) + return + if GLOBAL_ARGS.patch != '': + shutil.copyfile(GLOBAL_ARGS.patch, os.path.join(mgt_path, 'product.data')) + + log.info(text('remain-disk-capacity', (safeline_path, humen_size(free_space(safeline_path))))) + + log.info(text('download-compose')) + if not save_file_from_url('https://'+DOMAIN+'/release/latest/compose.yaml',os.path.join(safeline_path, 'docker-compose.yaml')): + log.error(text('fail-to-download-compose')) + return + rename_file(os.path.join(safeline_path, 'compose.yaml'),os.path.join(safeline_path, 'compose.yaml.bak')) + + mgt_port = generate_config_and_run(safeline_path) + if mgt_port is None: + return + + log.info(text('install-finish', PRODUCT)) + finish(mgt_port) + +def get_installed_dir(): + safeline_path = '' + safeline_path_proc = exec_command('docker','inspect','--format','\'{{index .Config.Labels "com.docker.compose.project.working_dir"}}\'', 'safeline-mgt') + if safeline_path_proc[0] == 0: + safeline_path = safeline_path_proc[1].strip().replace("'",'') + else: + log.debug("get installed dir error: "+ safeline_path_proc[2]) + log.debug("find safeline installed path: " + safeline_path) + if safeline_path == '' or not os.path.exists(safeline_path): + log.warning(text('fail-to-get-installed-dir')) + return ui_read(text('input-target-path', PRODUCT),None) + + return safeline_path + +def save_file_from_url(url, path): + log.debug('saving '+url+' to '+path) + data = get_url(url) + if data is None: + return False + with open(path, 'w') as f: + f.write(data) + return True + +def rename_file(src, dst): + if os.path.exists(src): + os.rename(src, dst) + +def remove_file(src): + if os.path.exists(src): + os.remove(src) + +def generate_config_and_run(safeline_path): + env_file = os.path.join(safeline_path, '.env') + env_bak_file = os.path.join(safeline_path, '.env.bak') + if os.path.exists(env_file): + shutil.copyfile(env_file, env_bak_file) + + try: + while True: + config = generate_config(safeline_path) + if docker_pull(safeline_path): + break + + pull_failed_prefix.append(config['IMAGE_PREFIX']) + log.info(text('try-another-image-source')) + + if not docker_up(safeline_path): + log.error(text('fail-to-up')) + rename_file(env_bak_file, env_file) + return None + except KeyboardInterrupt: + log.warning(text('keyboard-interrupt')) + rename_file(env_bak_file, env_file) + return None + except Exception as e: + log.error('start WAF failed: '+str(e)) + rename_file(env_bak_file, env_file) + return None + + remove_file(env_bak_file) + return config['MGT_PORT'] + +def upgrade(): + safeline_path = get_installed_dir() + + if not precheck_docker_compose() or not precheck_dns_scope(): + log.error(text('precheck-failed', PRODUCT)) + return + + log.info(text('download-compose')) + rename_file(os.path.join(safeline_path, 'compose.yaml'), os.path.join(safeline_path, 'compose.yaml.bak')) + if not save_file_from_url('https://'+DOMAIN+'/release/latest/compose.yaml', os.path.join(safeline_path, 'docker-compose.yaml')): + log.error(text('fail-to-download-compose')) + return + + mgt_port = generate_config_and_run(safeline_path) + if mgt_port is None: + return + + if GLOBAL_ARGS.image_clean: + image_clean() + + log.info(text('upgrade-finish', PRODUCT)) + finish(mgt_port) + +def finish(mgt_port): + init_mgt() + show_address(mgt_port) + +def reset_tengine(): + safeline_path = get_installed_dir() + resources_path = os.path.join(safeline_path, 'resources') + nginx_path = os.path.join(resources_path,'nginx') + if not os.path.exists(nginx_path): + log.error(text('fail-to-find-nginx')) + return + backup_path = os.path.join(resources_path, 'nginx.'+str(datetime.datetime.now().timestamp())) + log.info(text('nginx-backup-dir') +': '+ backup_path) + try: + shutil.move(nginx_path, backup_path) + except Exception as e: + log.error(text('fail-to-backup-nginx')+': '+str(e)) + return + + if docker_restart('safeline-tengine'): + docker_exec('safeline-mgt', 'gentenginewebsite') + + if os.path.exists(os.path.join(backup_path, 'static')): + try: + shutil.copy(os.path.join(backup_path, 'static'), os.path.join(nginx_path, 'static')) + except Exception as e: + log.error(text('fail-to-recover-static')+': '+str(e)) + return + + log.info(text('reset-tengine-finish')) + +def docker_restart_all(cwd): + if not docker_down(cwd): + log.error(text('fail-to-down')) + return False + + if not docker_up(cwd): + log.error(text('fail-to-up')) + return False + + return True + +def reset_postgres(): + safeline_path = get_installed_dir() + + env_file = os.path.join(safeline_path, '.env') + if not os.path.exists(env_file): + log.error(text('fail-to-find-env')) + return + + config = {} + read_config(env_file, config) + config['POSTGRES_PASSWORD'] = ''.join([random.choice(string.ascii_letters + string.digits) for i in range(20)]) + write_config(env_file, config) + + if not docker_exec('safeline-pg','psql -U safeline-ce -c "ALTER USER \\"safeline-ce\\" WITH PASSWORD \''+config['POSTGRES_PASSWORD']+'\';"'): + log.error(text('fail-to-reset-postgres-password')) + return + + action = ui_choice(text('if-restart-docker', PRODUCT), [ + ('y', text('yes')), + ('n', text('no')), + ]) + + if action.lower() == 'y': + if not precheck_docker_compose(): + log.error(text('precheck-failed', PRODUCT)) + return + + if not docker_restart_all(safeline_path): + return + + log.info(text('reset-postgres-password-finish')) + +def repair(): + action = ui_choice(text('choice-action'),[ + ('1', text('reset-tengine')), + ('2', text('reset-postgres')), + ]) + + if action =='1': + reset_tengine() + elif action =='2': + reset_postgres() + +def restart(): + safeline_path = get_installed_dir() + + if not precheck_docker_compose(): + log.error(text('precheck-failed', PRODUCT)) + return + + if not docker_restart_all(safeline_path): + return + + log.info(text('restart-docker-finish', PRODUCT)) + +def backup(): + pass + +def uninstall(): + safeline_path = get_installed_dir() + + action = ui_choice(text('if-remove-waf', (PRODUCT, safeline_path)),[ + ('y', text('yes')), + ('n', text('no')), + ]) + + if action == 'n': + return + + if not precheck_docker_compose(): + log.error(text('precheck-failed', PRODUCT)) + return + + if not docker_down(safeline_path): + log.error(text('fail-to-docker-down')) + return + + image_clean() + + try: + shutil.rmtree(safeline_path) + except Exception as e: + log.debug("remove dir failed: "+str(e)) + log.error(text('fail-to-remove-dir', PRODUCT)) + + log.info(text('uninstall-finish', PRODUCT)) + +ACTION = '' + +def get_version_from_input(old_version): + while True: + version = ui_read(text('input-target-version', ACTION),None) + if not check_version_format(version): + log.warning(text('version-format-error', version)) + continue + if not compare_version(old_version, version): + log.warning(text('can-not-downgrade', (PRODUCT, old_version, version))) + continue + return version.lstrip('v') + +def check_version_format(version): + if GLOBAL_ARGS.lts and not version.endswith('-lts'): + log.debug('lts version should end with -lts') + return False + split_version = version.lstrip('v').rstrip('-lts').split('.') + if len(split_version) != 3: + log.debug('split version length is not 3') + return False + try: + for v in split_version: + int(v) + except ValueError as e: + log.debug('check version %s format failed: %s' % (version, str(e))) + return False + return True + +def compare_version(old_version, new_version): + if old_version == 'latest': + try: + log.info(text('get-version-from-mgt')) + old_version = get_version_from_mgt() + except Exception as e: + log.debug('get version from mgt failed: '+str(e)) + log.warning(text('fail-to-get-version-from-mgt')) + log.warning(text('skip-version-compare', PRODUCT)) + return True + elif old_version == '': + return True + + if not check_version_format(old_version): + log.warning(text('version-format-error', old_version)) + log.warning(text('skip-version-compare', PRODUCT)) + return True + + split_old_version = old_version.lstrip('v').rstrip('-lts').split('.') + split_new_version = new_version.lstrip('v').rstrip('-lts').split('.') + + for index in range(len(split_old_version)): + int_old_version = int(split_old_version[index]) + int_new_version = int(split_new_version[index]) + if int_old_version > int_new_version: + return False + elif int_old_version < int_new_version: + return True + + return True + + +def get_version_from_mgt(): + proc = exec_command('docker exec safeline-mgt /app/mgt version',shell=True) + if proc[0] != 0: + raise Exception('stderr: ' + proc[2]) + for line in proc[1].split('\n'): + strip_line = line.strip() + if not strip_line.startswith('version'): + continue + return strip_line.lstrip('version').strip() + + raise Exception('mgt version not found') + +TARGET_VERSION = '' + +def get_version(old_version): + global TARGET_VERSION + if TARGET_VERSION != '': + return TARGET_VERSION + + log.info(text('get-version', PRODUCT)) + + # Try to read version from local version.json first + local_version_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'version.json') + try: + if os.path.exists(local_version_file): + with open(local_version_file, 'r') as f: + version = json.loads(f.read()) + if GLOBAL_ARGS.lts: + latest_version = version.get('lts_version', '') + else: + latest_version = version.get('latest_version', '') + if latest_version and check_version_format(latest_version): + if not compare_version(old_version, latest_version): + log.warning(text('can-not-downgrade', (old_version, latest_version))) + TARGET_VERSION = get_version_from_input(old_version) + else: + TARGET_VERSION = latest_version.lstrip('v') + return TARGET_VERSION + except Exception as e: + log.debug('read local version.json failed: %s' % str(e)) + + # Fall back to manual input + TARGET_VERSION = get_version_from_input(old_version) + return TARGET_VERSION + +def main(): + if SELF: + banner() + log.info(text('hello1')) + log.info(text('hello2')) + print() + + if GLOBAL_ARGS.lts: + log.info(text('install-channel', text('lts-release'))) + + if sys.version_info.major == 2 or (sys.version_info.major == 3 and sys.version_info.minor <= 5): + log.error(text('python-version-too-low')) + return + + if not sys.stdin.isatty(): + log.error(text('not-a-tty')) + return + + if os.geteuid() != 0: + log.error(text('not-root')) + return + + if platform.system() != 'Linux': + log.error(text('not-linux', (PRODUCT, platform.system()))) + return + + if platform.machine() not in ('aarch64', 'x86_64', 'AMD64'): + log.error(text('unsupported-arch', (PRODUCT, platform.machine()))) + return + + + action = ui_choice(text('choice-action'), [ + ('1', text('install')), + ('2', text('upgrade')), + ('3', text('uninstall')), + ('4', text('repair')), + ('5', text('restart')), + # ('4', text('backup')) + ]) + + global ACTION + if action == '1': + ACTION = text('install') + install() + elif action == '2': + ACTION = text('upgrade') + upgrade() + elif action == '3': + uninstall() + elif action == '4': + repair() + elif action == '5': + restart() + # elif action == '4': + # backup() + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + log.warning(text('keyboard-interrupt')) + pass + except Exception as e: + log.error(e) + finally: + if SELF: + print(color(text('talking-group') + '\n', [GREEN])) diff --git a/sdk/ingress-nginx/README.md b/sdk/ingress-nginx/README.md new file mode 100644 index 0000000..ecdd00c --- /dev/null +++ b/sdk/ingress-nginx/README.md @@ -0,0 +1,138 @@ +# ingress-nginx-safeline + +[Ingress-nginx](https://kubernetes.github.io/ingress-nginx/) plugin for Chaitin SafeLine Web Application Firewall (WAF). This plugin is used to protect your API from malicious requests. It can be used to block requests that contain malicious content in the request body, query parameters, headers, or URI. + +## Safeline Prepare +The detection engine of the SafeLine provides services by default via Unix socket. We need to modify it to use TCP, so it can be called by the t1k plugin. + +1.Navigate to the configuration directory of the SafeLine detection engine: +```shell +cd /data/safeline/resources/detector/ +``` +2.Open the `detector.yml` file in a text editor. Modify the bind configuration from Unix socket to TCP by adding the following settings: +```yaml +bind_addr: 0.0.0.0 +listen_port: 8000 +``` +These configuration values will override the default settings in the container, making the SafeLine engine listen on port 8000. + +3.Next, map the container’s port 8000 to the host machine. First, navigate to the SafeLine installation directory: +```shell +cd /data/safeline +``` + +4.Open the compose.yaml file in a text editor and add the ports field to the detector container to expose port 8000: +```yaml +... +detect: + ports: + - 8000:8000 +... +``` + +5.Save the changes and restart SafeLine with the following commands: +```shell +docker-compose down +docker-compose up -d +``` +This will apply the changes and activate the new configuration. + +## Plugin Usage + +### Step 1: Install the plugin + +way 1: Build your own ingress-nginx/controller image with the plugin installed. + +```dockerfile +FROM registry.k8s.io/ingress-nginx/controller:v1.10.1 + +USER root + +RUN apk add --no-cache make gcc unzip wget + +# install luaroncks +RUN wget https://luarocks.org/releases/luarocks-3.11.0.tar.gz && \ + tar zxpf luarocks-3.11.0.tar.gz && \ + cd luarocks-3.11.0 && \ + ./configure && \ + make && \ + make install && \ + cd .. && \ + rm -rf luarocks-3.11.0 luarocks-3.11.0.tar.gz + +RUN luarocks install ingress-nginx-safeline && \ + ln -s /usr/local/share/lua/5.1/safeline /etc/nginx/lua/plugins/safeline + +USER www-data +``` + +way 2: Use the chaitin ingress-nginx-controller image. + +replace image ingress-nginx-controller with `docker.io/chaitin/ingress-nginx-controller:v1.10.1` in your deployment. + +### Step 2: Configure the plugin + +use a ConfigMap to configure the plugin + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: safeline + namespace: default +data: + host: "YOUR_DETECTOR_HOST" + port: "YOUR_DETECTOR_PORT" +``` + +### Step 3: Configure the ingress-controller + +inject env `SAFELINE_HOST` and `SAFELINE_PORT` to the ingress-controller deployment + +```yaml +... +env: + - name: SAFELINE_HOST + valueFrom: + configMapKeyRef: + name: safeline + key: host + - name: SAFELINE_PORT + valueFrom: + configMapKeyRef: + name: safeline + key: port +... + +``` + +### Step 3: Enable the plugin + +enable safeline plugin in configmap + +```yaml +apiVersion: v1 +data: + allow-snippet-annotations: "false" + plugins: "safeline" +kind: ConfigMap +metadata: + name: ingress-nginx-controller + namespace: default +``` + +### Step 4: Set externalTrafficPolicy to Local +by default, the ingress-nginx-controller service is of type LoadBalancer, which means the source IP of the request will be the IP of the LoadBalancer. If you want to get the real source IP, you can set the externalTrafficPolicy to Local. + +### Step 5: Test the plugin + +use a simple http sql injection test + +```bash +curl -X POST http://localhost/ -d "select * from users where id=1 or 1=1" +``` + +you should get a 403 response. +```bash +{"code": 403, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "b53eb5b95796475699c52a019abb8e6a"} +``` \ No newline at end of file diff --git a/sdk/ingress-nginx/ingress-nginx-safeline-1.0.2-1.rockspec b/sdk/ingress-nginx/ingress-nginx-safeline-1.0.2-1.rockspec new file mode 100644 index 0000000..023d798 --- /dev/null +++ b/sdk/ingress-nginx/ingress-nginx-safeline-1.0.2-1.rockspec @@ -0,0 +1,20 @@ +package = "ingress-nginx-safeline" +version = "1.0.2-1" +source = { + url = "git://github.com/xbingW/ingress-nginx-safeline.git" +} +description = { + summary = "Ingress-Nginx plugin for Chaitin SafeLine Web Application Firewall", + homepage = "https://github.com/xbingW/ingress-nginx-safeline", + license = "Apache License 2.0", + maintainer = "Xiaobing Wang " +} +dependencies = { + "lua-resty-t1k" +} +build = { + type = "builtin", + modules = { + ["safeline.main"] = "lib/safeline/main.lua" + } +} diff --git a/sdk/ingress-nginx/ingress-nginx-safeline-1.0.3-1.rockspec b/sdk/ingress-nginx/ingress-nginx-safeline-1.0.3-1.rockspec new file mode 100644 index 0000000..aad3aa3 --- /dev/null +++ b/sdk/ingress-nginx/ingress-nginx-safeline-1.0.3-1.rockspec @@ -0,0 +1,20 @@ +package = "ingress-nginx-safeline" +version = "1.0.3-1" +source = { + url = "git://github.com/xbingW/ingress-nginx-safeline.git" +} +description = { + summary = "Ingress-Nginx plugin for Chaitin SafeLine Web Application Firewall", + homepage = "https://github.com/xbingW/ingress-nginx-safeline", + license = "Apache License 2.0", + maintainer = "Xiaobing Wang " +} +dependencies = { + "lua-resty-t1k >= 1.1.5" +} +build = { + type = "builtin", + modules = { + ["safeline.main"] = "lib/safeline/main.lua" + } +} diff --git a/sdk/ingress-nginx/ingress-nginx-safeline-1.0.4-1.rockspec b/sdk/ingress-nginx/ingress-nginx-safeline-1.0.4-1.rockspec new file mode 100644 index 0000000..5a59b3f --- /dev/null +++ b/sdk/ingress-nginx/ingress-nginx-safeline-1.0.4-1.rockspec @@ -0,0 +1,20 @@ +package = "ingress-nginx-safeline" +version = "1.0.4-1" +source = { + url = "git://github.com/chaitin/ingress-nginx-safeline.git" +} +description = { + summary = "Ingress-Nginx plugin for Chaitin SafeLine Web Application Firewall", + homepage = "https://github.com/chaitin/ingress-nginx-safeline", + license = "Apache License 2.0", + maintainer = "Xiaobing Wang " +} +dependencies = { + "lua-resty-t1k >= 1.1.5" +} +build = { + type = "builtin", + modules = { + ["safeline.main"] = "lib/safeline/main.lua" + } +} diff --git a/sdk/ingress-nginx/lib/safeline/main.lua b/sdk/ingress-nginx/lib/safeline/main.lua new file mode 100644 index 0000000..b474181 --- /dev/null +++ b/sdk/ingress-nginx/lib/safeline/main.lua @@ -0,0 +1,61 @@ +local t1k = require "resty.t1k" +local t1k_constants = require "resty.t1k.constants" + +local ngx = ngx +local fmt = string.format + +local blocked_message = [[{"code": %s, "success":false, ]] .. + [["message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "%s"}]] + +local _M = {} + +local mode = os.getenv("SAFELINE_MODE") +local host = os.getenv("SAFELINE_HOST") +local port = os.getenv("SAFELINE_PORT") +local connect_timeout = os.getenv("SAFELINE_CONNECT_TIMEOUT") +local send_timeout = os.getenv("SAFELINE_SEND_TIMEOUT") +local read_timeout = os.getenv("SAFELINE_READ_TIMEOUT") +local req_body_size = os.getenv("SAFELINE_REQ_BODY_SIZE") +local keepalive_size = os.getenv("SAFELINE_KEEPALIVE_SIZE") +local keepalive_timeout = os.getenv("SAFELINE_KEEPALIVE_TIMEOUT") +local remote_addr = os.getenv("SAFELINE_REMOTE_ADDR") + +local function get_conf() + local t = { + mode = mode or "block", + host = host, + port = port, + connect_timeout = connect_timeout or 1000, + send_timeout = send_timeout or 1000, + read_timeout = read_timeout or 1000, + req_body_size = req_body_size or 1024, + keepalive_size = keepalive_size or 256, + keepalive_timeout = keepalive_timeout or 60000, + remote_addr = remote_addr + } + return t +end + +function _M.rewrite() + local t = get_conf() + if not t.host then + ngx.log(ngx.ERR, "safeline host is required") + return + end + local ok, err, result = t1k.do_access(t, false) + if not ok then + ngx.log(ngx.ERR, "failed to detector req: ", err) + return + end + if result then + if result.action == t1k_constants.ACTION_BLOCKED then + local msg = fmt(blocked_message, result.status, result.event_id) + ngx.log(ngx.ERR, "blocked by safeline waf: ",msg) + ngx.status = tonumber(result.status,10) + ngx.say(msg) + return ngx.exit(ngx.HTTP_OK) + end + end +end + +return _M \ No newline at end of file diff --git a/sdk/kong/Readme.md b/sdk/kong/Readme.md new file mode 100644 index 0000000..67d134e --- /dev/null +++ b/sdk/kong/Readme.md @@ -0,0 +1,69 @@ +# Kong Safeline Plugin +Kong plugin for Chaitin SafeLine Web Application Firewall (WAF). This plugin is used to protect your API from malicious requests. It can be used to block requests that contain malicious content in the request body, query parameters, headers, or URI. + +## Safeline Prepare +The detection engine of the SafeLine provides services by default via Unix socket. We need to modify it to use TCP, so it can be called by the t1k plugin. + +1.Navigate to the configuration directory of the SafeLine detection engine: +```shell +cd /data/safeline/resources/detector/ +``` +2.Open the `detector.yml` file in a text editor. Modify the bind configuration from Unix socket to TCP by adding the following settings: +```yaml +bind_addr: 0.0.0.0 +listen_port: 8000 +``` +These configuration values will override the default settings in the container, making the SafeLine engine listen on port 8000. + +3.Next, map the container’s port 8000 to the host machine. First, navigate to the SafeLine installation directory: +```shell +cd /data/safeline +``` + +4.Open the compose.yaml file in a text editor and add the ports field to the detector container to expose port 8000: +```yaml +... +detect: + ports: + - 8000:8000 +... +``` + +5.Save the changes and restart SafeLine with the following commands: +```shell +docker-compose down +docker-compose up -d +``` +This will apply the changes and activate the new configuration. + +## Plugin Installation +To install the plugin, run the following command in your Kong server: + +```shell +$ luarocks install kong-safeline +``` + +## Plugin Configuration +You can add the plugin to your API by making the following request: + +```shell +# if your detector is running on tcp port +$ curl -X POST http://kong:8001/services/{name}/plugins \ + --data "name=safeline" \ + --data "config.host=your_detector_host" \ + --data "config.port=your_detector_port" +# if your detector is running on unix socket +$ curl -X POST http://kong:8001/services/{name}/plugins \ + --data "name=safeline" \ + --data "config.host=unix:/path/to/your/unix/socket" +``` + +## Test +You can test the plugin by sending a request to your API with malicious content. If the request is blocked, you will receive a `403 Forbidden` response. + +```shell +$ curl -X POST http://kong:8000?1=1%20and%202=2 + +# you will receive a 403 Forbidden response +{"code": 403, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "8b41a021ea9541c89bb88f3773b4da24"} +``` diff --git a/sdk/kong/kong-safeline-1.0.0-1.rockspec b/sdk/kong/kong-safeline-1.0.0-1.rockspec new file mode 100644 index 0000000..a6bf23d --- /dev/null +++ b/sdk/kong/kong-safeline-1.0.0-1.rockspec @@ -0,0 +1,21 @@ +package = "kong-safeline" +version = "1.0.0-1" +source = { + url = "git://github.com/xbingW/kong-safeline.git" +} +description = { + summary = "Kong plugin for Chaitin SafeLine Web Application Firewall", + homepage = "https://github.com/xbingW/kong-safeline", + license = "Apache License 2.0", + maintainer = "Xiaobing Wang " +} +dependencies = { + "lua-resty-t1k" +} +build = { + type = "builtin", + modules = { + ["kong.plugins.safeline.handler"] = "kong/plugins/safeline/handler.lua", + ["kong.plugins.safeline.schema"] = "kong/plugins/safeline/schema.lua" + } +} diff --git a/sdk/kong/kong-safeline-1.0.1-1.rockspec b/sdk/kong/kong-safeline-1.0.1-1.rockspec new file mode 100644 index 0000000..623d881 --- /dev/null +++ b/sdk/kong/kong-safeline-1.0.1-1.rockspec @@ -0,0 +1,21 @@ +package = "kong-safeline" +version = "1.0.1-1" +source = { + url = "git://github.com/xbingW/kong-safeline.git" +} +description = { + summary = "Kong plugin for Chaitin SafeLine Web Application Firewall", + homepage = "https://github.com/xbingW/kong-safeline", + license = "Apache License 2.0", + maintainer = "Xiaobing Wang " +} +dependencies = { + "lua-resty-t1k" +} +build = { + type = "builtin", + modules = { + ["kong.plugins.safeline.handler"] = "kong/plugins/safeline/handler.lua", + ["kong.plugins.safeline.schema"] = "kong/plugins/safeline/schema.lua" + } +} diff --git a/sdk/kong/kong-safeline-1.0.2-1.rockspec b/sdk/kong/kong-safeline-1.0.2-1.rockspec new file mode 100644 index 0000000..9a09c38 --- /dev/null +++ b/sdk/kong/kong-safeline-1.0.2-1.rockspec @@ -0,0 +1,21 @@ +package = "kong-safeline" +version = "1.0.2-1" +source = { + url = "git://github.com/xbingW/kong-safeline.git" +} +description = { + summary = "Kong plugin for Chaitin SafeLine Web Application Firewall", + homepage = "https://github.com/xbingW/kong-safeline", + license = "Apache License 2.0", + maintainer = "Xiaobing Wang " +} +dependencies = { + "lua-resty-t1k >= 1.1.5" +} +build = { + type = "builtin", + modules = { + ["kong.plugins.safeline.handler"] = "kong/plugins/safeline/handler.lua", + ["kong.plugins.safeline.schema"] = "kong/plugins/safeline/schema.lua" + } +} diff --git a/sdk/kong/kong-safeline-1.0.3-1.rockspec b/sdk/kong/kong-safeline-1.0.3-1.rockspec new file mode 100644 index 0000000..1ce7c5a --- /dev/null +++ b/sdk/kong/kong-safeline-1.0.3-1.rockspec @@ -0,0 +1,31 @@ +package = "kong-safeline" +version = "1.0.3-1" +source = { + url = "file://kong-safeline-1.0.3.tar.gz", +} +build = { + type = "script", + rockspec = { + build = { + "git clone https://github.com/chaitin/SafeLine.git", + "cp -r sdk/kong .", + "rm -rf SafeLine" + } + } +} +description = { + summary = "Kong plugin for Chaitin SafeLine Web Application Firewall", + homepage = "https://github.com/chaitin/SafeLine", + license = "Apache License 2.0", + maintainer = "Xiaobing Wang " +} +dependencies = { + "lua-resty-t1k >= 1.1.5" +} +build = { + type = "builtin", + modules = { + ["kong.plugins.safeline.handler"] = "kong/plugins/safeline/handler.lua", + ["kong.plugins.safeline.schema"] = "kong/plugins/safeline/schema.lua" + } +} diff --git a/sdk/kong/kong-safeline-1.0.4-1.rockspec b/sdk/kong/kong-safeline-1.0.4-1.rockspec new file mode 100644 index 0000000..2e6719e --- /dev/null +++ b/sdk/kong/kong-safeline-1.0.4-1.rockspec @@ -0,0 +1,31 @@ +package = "kong-safeline" +version = "1.0.4-1" +source = { + url = "git://github.com/chaitin/SafeLine.git", +} +build = { + type = "script", + rockspec = { + build = { + "git clone https://github.com/chaitin/SafeLine.git", + "cp -r sdk/kong .", + "rm -rf SafeLine" + } + } +} +description = { + summary = "Kong plugin for Chaitin SafeLine Web Application Firewall", + homepage = "https://github.com/chaitin/SafeLine", + license = "Apache License 2.0", + maintainer = "Xiaobing Wang " +} +dependencies = { + "lua-resty-t1k >= 1.1.5" +} +build = { + type = "builtin", + modules = { + ["kong.plugins.safeline.handler"] = "kong/plugins/safeline/handler.lua", + ["kong.plugins.safeline.schema"] = "kong/plugins/safeline/schema.lua" + } +} diff --git a/sdk/kong/kong-safeline-1.0.5-1.rockspec b/sdk/kong/kong-safeline-1.0.5-1.rockspec new file mode 100644 index 0000000..946808d --- /dev/null +++ b/sdk/kong/kong-safeline-1.0.5-1.rockspec @@ -0,0 +1,31 @@ +package = "kong-safeline" +version = "1.0.5-1" +source = { + url = "git://github.com/chaitin/SafeLine.git", +} +build = { + type = "script", + rockspec = { + build = { + "git clone https://github.com/chaitin/SafeLine.git", + "cp -r sdk/kong .", + "rm -rf SafeLine" + } + } +} +description = { + summary = "Kong plugin for Chaitin SafeLine Web Application Firewall", + homepage = "https://github.com/chaitin/SafeLine", + license = "Apache License 2.0", + maintainer = "Xiaobing Wang " +} +dependencies = { + "lua-resty-t1k >= 1.1.5" +} +build = { + type = "builtin", + modules = { + ["kong.plugins.safeline.handler"] = "kong/plugins/safeline/handler.lua", + ["kong.plugins.safeline.schema"] = "kong/plugins/safeline/schema.lua" + } +} diff --git a/sdk/kong/kong-safeline-1.0.6-1.rockspec b/sdk/kong/kong-safeline-1.0.6-1.rockspec new file mode 100644 index 0000000..3393e3a --- /dev/null +++ b/sdk/kong/kong-safeline-1.0.6-1.rockspec @@ -0,0 +1,21 @@ +package = "kong-safeline" +version = "1.0.6-1" +source = { + url = "git://github.com/xbingW/kong-safeline.git", +} +description = { + summary = "Kong plugin for Chaitin SafeLine Web Application Firewall", + homepage = "https://github.com/chaitin/SafeLine", + license = "Apache License 2.0", + maintainer = "Xiaobing Wang " +} +dependencies = { + "lua-resty-t1k >= 1.1.5" +} +build = { + type = "builtin", + modules = { + ["kong.plugins.safeline.handler"] = "kong/plugins/safeline/handler.lua", + ["kong.plugins.safeline.schema"] = "kong/plugins/safeline/schema.lua" + } +} diff --git a/sdk/kong/kong-safeline-1.0.7-1.rockspec b/sdk/kong/kong-safeline-1.0.7-1.rockspec new file mode 100644 index 0000000..d68e96b --- /dev/null +++ b/sdk/kong/kong-safeline-1.0.7-1.rockspec @@ -0,0 +1,21 @@ +package = "kong-safeline" +version = "1.0.7-1" +source = { + url = "git://github.com/chaitin/kong-safeline.git", +} +description = { + summary = "Kong plugin for Chaitin SafeLine Web Application Firewall", + homepage = "https://github.com/chaitin/SafeLine", + license = "Apache License 2.0", + maintainer = "Xiaobing Wang " +} +dependencies = { + "lua-resty-t1k >= 1.1.5" +} +build = { + type = "builtin", + modules = { + ["kong.plugins.safeline.handler"] = "kong/plugins/safeline/handler.lua", + ["kong.plugins.safeline.schema"] = "kong/plugins/safeline/schema.lua" + } +} diff --git a/sdk/kong/kong/plugins/safeline/handler.lua b/sdk/kong/kong/plugins/safeline/handler.lua new file mode 100644 index 0000000..06da7bf --- /dev/null +++ b/sdk/kong/kong/plugins/safeline/handler.lua @@ -0,0 +1,47 @@ +local t1k = require "resty.t1k" +local t1k_constants = require "resty.t1k.constants" + +local fmt = string.format + +local SafelineHandler = { + VERSION = "0.0.1", + PRIORITY = 1000 +} + +local blocked_message = [[{"code": %s, "success":false, ]] .. + [["message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "%s"}]] + +local function get_conf(conf) + local t = { + mode = conf.mode, + host = conf.host, + port = conf.port, + connect_timeout = conf.connect_timeout, + send_timeout = conf.send_timeout, + read_timeout = conf.read_timeout, + req_body_size = conf.req_body_size, + keepalive_size = conf.keepalive_size, + keepalive_timeout = conf.keepalive_timeout, + remote_addr = conf.remote_addr + } + return t +end + +function SafelineHandler:access(conf) + -- your custom code here + local t = get_conf(conf) + local ok, err, result = t1k.do_access(t, false) + if not ok then + kong.log.err("failed to detector req: ", err) + end + if result and result.status then + if result.action == t1k_constants.ACTION_BLOCKED then + local msg = fmt(blocked_message, result.status, result.event_id) + kong.log.debug("blocked by safeline: ",msg) + return kong.response.exit(tonumber(result.status), msg) + end + end +end + + +return SafelineHandler diff --git a/sdk/kong/kong/plugins/safeline/schema.lua b/sdk/kong/kong/plugins/safeline/schema.lua new file mode 100644 index 0000000..9d096f0 --- /dev/null +++ b/sdk/kong/kong/plugins/safeline/schema.lua @@ -0,0 +1,73 @@ +local typedefs = require "kong.db.schema.typedefs" + +return { + name = "kong-safeline", + fields = {{ + consumer = typedefs.no_consumer + }, { + protocols = typedefs.protocols_http + }, { + config = { + type = "record", + fields = {{ + host = { + type = "string", + required = true + } + }, { + port = { + type = "number", + required = false + } + }, { + mode = { + type = "string", + required = false, + default = "block", + one_of = {"monitor", "block", "off"} + } + }, { + connect_timeout = { + type = "number", + required = false, + default = 1000 + } + }, { + send_timeout = { + type = "number", + required = false, + default = 1000 + } + }, { + read_timeout = { + type = "number", + required = false, + default = 1000 + } + }, { + req_body_size = { + type = "number", + required = false, + default = 1000 + } + }, { + keepalive_size = { + type = "number", + required = false, + default = 1000 + } + }, { + keepalive_timeout = { + type = "number", + required = false, + default = 1000 + } + }, { + remote_addr = { + type = "string", + required = false + } + }} + } + }} +} diff --git a/sdk/lua-resty-t1k/.github/workflows/release.yml b/sdk/lua-resty-t1k/.github/workflows/release.yml new file mode 100644 index 0000000..b86272c --- /dev/null +++ b/sdk/lua-resty-t1k/.github/workflows/release.yml @@ -0,0 +1,46 @@ +name: Release + +on: + push: + tags: + - "v*" + +jobs: + release: + name: Release + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install Lua + uses: leafo/gh-actions-lua@v10 + + - name: Install Luarocks + uses: leafo/gh-actions-luarocks@v4 + + - name: Extract release tag + id: release_tag + run: | + # Extract the tag name from the ref + tag="${GITHUB_REF#refs/tags/}" + version_without_v="${tag#v}" + echo "version=${tag}" >> $GITHUB_ENV + echo "version_without_v=${version_without_v}" >> $GITHUB_ENV + + - name: Create Release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ env.version }} + release_name: ${{ env.version }} + draft: false + prerelease: false + + - name: Upload to luarocks + env: + LUAROCKS_TOKEN: ${{ secrets.LUAROCKS_TOKEN }} + run: | + luarocks install dkjson + luarocks upload rockspec/lua-resty-t1k-${{ env.version_without_v }}-0.rockspec --api-key=${{ secrets.LUAROCKS_API_KEY }} diff --git a/sdk/lua-resty-t1k/.github/workflows/test.yml b/sdk/lua-resty-t1k/.github/workflows/test.yml new file mode 100644 index 0000000..d5ceeb1 --- /dev/null +++ b/sdk/lua-resty-t1k/.github/workflows/test.yml @@ -0,0 +1,63 @@ +name: Test + +on: [ push, pull_request ] + +jobs: + luacheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: leafo/gh-actions-lua@v10 + with: + luaVersion: "luajit-openresty" + - uses: leafo/gh-actions-luarocks@v4 + - run: luarocks install luacheck + - run: luacheck lib + + run_tests: + strategy: + matrix: + openresty_version: + - 1.17.8.2 + - 1.19.9.1 + - 1.21.4.3 + - 1.25.3.1 + + services: + detector: + image: chaitin/safeline-detector:t1k-ci-1.6.0 + + runs-on: ubuntu-latest + container: + image: openresty/openresty:${{ matrix.openresty_version }}-alpine-fat + # --init runs tinit as PID 1 and prevents the 'WARNING: killing the child process' spam from the test suite + options: --init + + steps: + - name: Install deps + run: | + apk add --no-cache bash bind-tools curl git git-lfs libarchive-tools perl perl-dev wget + ln -s /usr/bin/bsdtar /usr/bin/tar + + - name: Install CPAN + run: curl -s -L http://xrl.us/cpanm > /bin/cpanm && chmod +x /bin/cpanm + + - name: Cache + uses: actions/cache@v3 + with: + path: | + ~/.cpan + ~/.cache + key: ${{ runner.os }}-${{ matrix.openresty_version }}-cache + + - name: Install Test::Nginx + run: cpanm -q -n Test::Nginx + + - uses: actions/checkout@v4 + with: + lfs: true + + - name: Run tests + run: | + curl -fs -X POST -H "Content-Type: application/octet-stream" --data-binary "@ci/bytecode" "http://detector:8001/update/policy" + env DETECTOR_IP=$(dig detector +short) prove -r t/ diff --git a/sdk/lua-resty-t1k/.gitignore b/sdk/lua-resty-t1k/.gitignore new file mode 100644 index 0000000..2864f68 --- /dev/null +++ b/sdk/lua-resty-t1k/.gitignore @@ -0,0 +1,25 @@ +# IntelliJ project files +.idea +*.iml +out +gen + +# Visual Studio Code +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +# Test::Nginx files +t/servroot + +# luarocks build files +*.src.rock diff --git a/sdk/lua-resty-t1k/.luacheckrc b/sdk/lua-resty-t1k/.luacheckrc new file mode 100644 index 0000000..3d0e861 --- /dev/null +++ b/sdk/lua-resty-t1k/.luacheckrc @@ -0,0 +1,6 @@ +std = "ngx_lua" +redefined = false +max_line_length = 130 +max_code_line_length = 130 +max_string_line_length = 130 +max_comment_line_length = 130 diff --git a/sdk/lua-resty-t1k/LICENSE b/sdk/lua-resty-t1k/LICENSE new file mode 100644 index 0000000..d6460f1 --- /dev/null +++ b/sdk/lua-resty-t1k/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2023 Beijing Chaitin Technology Co., Ltd. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/sdk/lua-resty-t1k/README.md b/sdk/lua-resty-t1k/README.md new file mode 100644 index 0000000..cda6d8e --- /dev/null +++ b/sdk/lua-resty-t1k/README.md @@ -0,0 +1,76 @@ +# lua-resty-t1k + +[![LuaRocks](https://img.shields.io/luarocks/v/blaisewang/lua-resty-t1k?style=flat-square)](https://luarocks.org/modules/blaisewang/lua-resty-t1k) +[![Releases](https://img.shields.io/github/v/release/chaitin/lua-resty-t1k?style=flat-square)](https://github.com/chaitin/lua-resty-t1k/releases) +[![License](https://img.shields.io/github/license/chaitin/lua-resty-t1k?color=ff69b4&style=flat-square)](https://github.com/chaitin/lua-resty-t1k/blob/main/LICENSE) + +## Name + +Lua implementation of the T1K protocol for [Chaitin/SafeLine](https://github.com/chaitin/safeline) Web Application Firewall. + +## Status + +Production ready. + +[![Test](https://img.shields.io/github/actions/workflow/status/chaitin/lua-resty-t1k/test.yml?logo=github&style=flat-square)](https://github.com/chaitin/lua-resty-t1k/actions) + +## Installation + +```bash +luarocks install lua-resty-t1k +``` + +If you are in Mainland China + +```bash +luarocks install lua-resty-t1k --server https://luarocks.cn +``` + +## Synopsis + +```lua +location / { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "block", -- block or monitor or off, default off + host = "unix:/workdir/snserver.sock", -- required, SafeLine WAF detection service host, unix domain socket, IP, or domain is supported, string + port = 8000, -- required when the host is an IP or domain, SafeLine WAF detection service port, integer + connect_timeout = 1000, -- connect timeout, in milliseconds, integer, default 1s (1000ms) + send_timeout = 1000, -- send timeout, in milliseconds, integer, default 1s (1000ms) + read_timeout = 1000, -- read timeout, in milliseconds, integer, default 1s (1000ms) + req_body_size = 1024, -- request body size, in KB, integer, default 1MB (1024KB) + keepalive_size = 256, -- maximum concurrent idle connections to the SafeLine WAF detection service, integer, default 256 + keepalive_timeout = 60000, -- idle connection timeout, in milliseconds, integer, default 60s (60000ms) + remote_addr = "http_x_forwarded_for: 1", -- remote address from ngx.var.VARIABLE, string, default from ngx.var.remote_addr + } + + local ok, err, _ = t1k.do_access(t, true) + if not ok then + ngx.log(ngx.ERR, err) + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } +} +``` + +## Lua Resty T1K vs. C T1K + +[C T1K](https://t1k.chaitin.com/), as part of SafeLine's enterprise edition, is a deployment mode crafted in C language for enhanced performance. +It is compatible with all versions of Nginx and does not require deployment via OpenResty (lua_nginx_module). + +| | Lua Resty T1K | C T1K | +|-----------------------|---------------|-------| +| Request Detection | ✅ | ✅ | +| Response Detection | ❌ | ✅ | +| Health Checks* | ❌ | ✅ | +| Cookie Protection | ❌ | ✅ | +| Bot Protection | ❌ | ✅ | +| Proxy-side Statistics | ❌ | ✅ | + +* APISIX implements health check functionality for the `chaitin-waf` plugin. For more information, please see the [chaitin-waf documentation](https://apisix.apache.org/docs/apisix/next/plugins/chaitin-waf/). diff --git a/sdk/lua-resty-t1k/ci/.dockerignore b/sdk/lua-resty-t1k/ci/.dockerignore new file mode 100644 index 0000000..91cf71a --- /dev/null +++ b/sdk/lua-resty-t1k/ci/.dockerignore @@ -0,0 +1 @@ +/bytecode diff --git a/sdk/lua-resty-t1k/ci/Dockerfile b/sdk/lua-resty-t1k/ci/Dockerfile new file mode 100644 index 0000000..d4ffe5f --- /dev/null +++ b/sdk/lua-resty-t1k/ci/Dockerfile @@ -0,0 +1,6 @@ +# Usage: +# docker build -t chaitin/safeline-detector:t1k-ci-1.6.0 . +# docker push chaitin/safeline-detector:t1k-ci-1.6.0 +FROM chaitin/safeline-detector:1.6.0 + +RUN sed -i "s/^# bind_addr/bind_addr/; s/^# listen_port/listen_port/; s/^bind_addr: unix/# bind_addr: unix/;" /detector/snserver.yml diff --git a/sdk/lua-resty-t1k/ci/bytecode b/sdk/lua-resty-t1k/ci/bytecode new file mode 100644 index 0000000..d207643 Binary files /dev/null and b/sdk/lua-resty-t1k/ci/bytecode differ diff --git a/sdk/lua-resty-t1k/lib/resty/t1k.lua b/sdk/lua-resty-t1k/lib/resty/t1k.lua new file mode 100644 index 0000000..15c9cd4 --- /dev/null +++ b/sdk/lua-resty-t1k/lib/resty/t1k.lua @@ -0,0 +1,97 @@ +local consts = require "resty.t1k.constants" +local filter = require "resty.t1k.filter" +local handler = require "resty.t1k.handler" +local log = require "resty.t1k.log" +local request = require "resty.t1k.request" +local utils = require "resty.t1k.utils" + +local lower = string.lower + +local ngx = ngx +local nlog = ngx.log + +local log_fmt = log.fmt +local debug_fmt = log.debug_fmt + +local _M = { + _VERSION = '1.0.0' +} + +local DEFAULT_T1K_CONNECT_TIMEOUT = 1000 -- 1s +local DEFAULT_T1K_SEND_TIMEOUT = 1000 -- 1s +local DEFAULT_T1K_READ_TIMEOUT = 1000 -- 1s +local DEFAULT_T1K_REQ_BODY_SIZE = 1024 -- 1024 KB +local DEFAULT_T1K_KEEPALIVE_SIZE = 256 +local DEFAULT_T1K_KEEPALIVE_TIMEOUT = 60 * 1000 -- 60s + +function _M.do_access(t, handle) + local ok, err, result + local opts = {} + t = t or {} + + if not t.mode then + return true, nil, nil + end + + opts.mode = lower(t.mode) + if opts.mode == consts.MODE_OFF then + nlog(debug_fmt("t1k is not enabled")) + return true, nil, nil + end + + if opts.mode ~= consts.MODE_OFF and opts.mode ~= consts.MODE_BLOCK and opts.mode ~= consts.MODE_MONITOR then + err = log_fmt("invalid t1k mode: %s", t.mode) + return nil, err, nil + end + + if not t.host then + err = log_fmt("invalid t1k host: %s", t.host) + return nil, err, nil + end + opts.host = t.host + + if utils.starts_with(opts.host, consts.UNIX_SOCK_PREFIX) then + opts.uds = true + else + if not tonumber(t.port) then + err = log_fmt("invalid t1k port: %s", t.port) + return nil, err, nil + end + opts.port = tonumber(t.port) + end + + opts.connect_timeout = t.connect_timeout or DEFAULT_T1K_CONNECT_TIMEOUT + opts.send_timeout = t.send_timeout or DEFAULT_T1K_SEND_TIMEOUT + opts.read_timeout = t.read_timeout or DEFAULT_T1K_READ_TIMEOUT + opts.req_body_size = t.req_body_size or DEFAULT_T1K_REQ_BODY_SIZE + opts.keepalive_size = t.keepalive_size or DEFAULT_T1K_KEEPALIVE_SIZE + opts.keepalive_timeout = t.keepalive_timeout or DEFAULT_T1K_KEEPALIVE_TIMEOUT + + if t.remote_addr then + local var, idx = utils.to_var_idx(t.remote_addr) + opts.remote_addr_var = var + opts.remote_addr_idx = idx + end + + ok, err, result = request.do_request(opts) + if not ok then + return ok, err, result + end + + if handle and opts.mode == consts.MODE_BLOCK then + ok, err = _M.do_handle(result) + end + + return ok, err, result +end + +function _M.do_handle(t) + local ok, err = handler.handle(t) + return ok, err +end + +function _M.do_header_filter() + filter.do_header_filter() +end + +return _M diff --git a/sdk/lua-resty-t1k/lib/resty/t1k/buffer.lua b/sdk/lua-resty-t1k/lib/resty/t1k/buffer.lua new file mode 100644 index 0000000..965e99d --- /dev/null +++ b/sdk/lua-resty-t1k/lib/resty/t1k/buffer.lua @@ -0,0 +1,24 @@ +local _M = { + _VERSION = '1.0.0', +} + +function _M:new(o) + o = o or {} + setmetatable(o, self) + self.__index = self + return o +end + +function _M:add(v) + self[#self + 1] = v +end + +function _M:len() + local len = 0 + for _, v in ipairs(self) do + len = len + #v + end + return len +end + +return _M diff --git a/sdk/lua-resty-t1k/lib/resty/t1k/constants.lua b/sdk/lua-resty-t1k/lib/resty/t1k/constants.lua new file mode 100644 index 0000000..0f4dff7 --- /dev/null +++ b/sdk/lua-resty-t1k/lib/resty/t1k/constants.lua @@ -0,0 +1,30 @@ +local t = {} + +t.ACTION_PASSED = "." +t.ACTION_BLOCKED = "?" + +t.MODE_OFF = "off" +t.MODE_BLOCK = "block" +t.MODE_MONITOR = "monitor" + +t.T1K_HEADER_SIZE = 5 + +t.TAG_HEAD = 0x01 +t.TAG_BODY = 0x02 +t.TAG_EXTRA = 0x03 +t.TAG_VERSION = 0x20 +t.TAG_EXTRA_HEADER = 0x23 +t.TAG_EXTRA_BODY = 0x24 + +t.MASK_FIRST = 0x40 +t.MASK_LAST = 0x80 + +t.NGX_HTTP_HEADER_PREFIX = "http_" + +t.BLOCK_CONTENT_TYPE = "application/json" +t.BLOCK_CONTENT_FORMAT = [[ +{"code": %s, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "%s"}]] + +t.UNIX_SOCK_PREFIX = "unix:" + +return t diff --git a/sdk/lua-resty-t1k/lib/resty/t1k/file.lua b/sdk/lua-resty-t1k/lib/resty/t1k/file.lua new file mode 100644 index 0000000..0ed1303 --- /dev/null +++ b/sdk/lua-resty-t1k/lib/resty/t1k/file.lua @@ -0,0 +1,34 @@ +local buffer = require "resty.t1k.buffer" + +local _M = { + _VERSION = '1.0.0' +} + +local buffer_size = 2 ^ 13 + +function _M.read(p, size) + size = (not size or size < 0) and 0 or size + + local f, err = io.open(p, "rb") + if not f or err then + return nil, err, nil + end + + local left = size + local buf = buffer:new() + + while left ~= 0 do + local block_size = math.min(left, buffer_size) + local block = f:read(block_size) + if not block then + break + end + buf:add(block) + left = math.max(left - block_size, 0) + end + f:close() + + return true, nil, buf +end + +return _M diff --git a/sdk/lua-resty-t1k/lib/resty/t1k/filter.lua b/sdk/lua-resty-t1k/lib/resty/t1k/filter.lua new file mode 100644 index 0000000..a192b3f --- /dev/null +++ b/sdk/lua-resty-t1k/lib/resty/t1k/filter.lua @@ -0,0 +1,50 @@ +local _M = { + _VERSION = '1.0.0' +} + +local find = string.find +local sub = string.sub + +local ngx = ngx + +local function parse_extra_header(extra_header) + local t = {} + local idx = 1 + + while (idx <= #extra_header) do + local key, val + local _, to = find(extra_header, ":", idx) + if to == 0 then + break + else + key = sub(extra_header, idx, to - 1) + end + + idx = to + 1 + _, to = find(extra_header, "\n", idx) + if to == 0 then + break + else + val = sub(extra_header, idx, to - 1) + end + + t[key] = val + idx = to + 1 + end + + return t +end + +function _M.do_header_filter() + local extra_header = ngx.ctx.t1k_extra_header + if extra_header ~= nil then + local header_table = parse_extra_header(extra_header) + for k, v in pairs(header_table) do + if k ~= nil and v ~= nil then + ngx.header[k] = v + end + end + end +end + +return _M diff --git a/sdk/lua-resty-t1k/lib/resty/t1k/handler.lua b/sdk/lua-resty-t1k/lib/resty/t1k/handler.lua new file mode 100644 index 0000000..e1176ea --- /dev/null +++ b/sdk/lua-resty-t1k/lib/resty/t1k/handler.lua @@ -0,0 +1,36 @@ +local consts = require "resty.t1k.constants" +local log = require "resty.t1k.log" + +local fmt = string.format + +local ngx = ngx + +local log_fmt = log.fmt + +local _M = { + _VERSION = '1.0.0' +} + +function _M.handle(t) + local t_type = type(t) + if t_type ~= "table" then + local err = log_fmt("invalid result type: %s", t_type) + return nil, err + end + + local action = t["action"] + if action == consts.ACTION_PASSED then + return true, nil + elseif action == consts.ACTION_BLOCKED then + ngx.status = t["status"] or ngx.HTTP_FORBIDDEN + ngx.header.content_type = consts.BLOCK_CONTENT_TYPE + ngx.say(fmt(consts.BLOCK_CONTENT_FORMAT, ngx.status, t["event_id"])) + + return ngx.exit(ngx.status) + else + local err = log_fmt("unknown action from t1k server: %s", action) + return nil, err + end +end + +return _M diff --git a/sdk/lua-resty-t1k/lib/resty/t1k/log.lua b/sdk/lua-resty-t1k/lib/resty/t1k/log.lua new file mode 100644 index 0000000..cc2a6ec --- /dev/null +++ b/sdk/lua-resty-t1k/lib/resty/t1k/log.lua @@ -0,0 +1,27 @@ +local _M = { + _VERSION = '1.0.0' +} + +local fmt = string.format + +local ERR = ngx.ERR +local WARN = ngx.WARN +local DEBUG = ngx.DEBUG + +function _M.fmt(formatstring, ...) + return fmt("lua-resty-t1k: " .. formatstring, ...) +end + +function _M.err_fmt(formatstring, ...) + return ERR, _M.fmt(formatstring, ...) +end + +function _M.warn_fmt(formatstring, ...) + return WARN, _M.fmt(formatstring, ...) +end + +function _M.debug_fmt(formatstring, ...) + return DEBUG, _M.fmt(formatstring, ...) +end + +return _M diff --git a/sdk/lua-resty-t1k/lib/resty/t1k/request.lua b/sdk/lua-resty-t1k/lib/resty/t1k/request.lua new file mode 100644 index 0000000..7cbbfd5 --- /dev/null +++ b/sdk/lua-resty-t1k/lib/resty/t1k/request.lua @@ -0,0 +1,365 @@ +local bit = require "bit" + +local buffer = require "resty.t1k.buffer" +local consts = require "resty.t1k.constants" +local file = require "resty.t1k.file" +local log = require "resty.t1k.log" +local utils = require "resty.t1k.utils" +local uuid = require "resty.t1k.uuid" + +local _M = { + _VERSION = '1.0.0', +} + +local bor = bit.bor +local byte = string.byte +local char = string.char +local fmt = string.format +local sub = string.sub +local concat = table.concat + +local ngx = ngx +local nlog = ngx.log +local ngx_req = ngx.req +local ngx_socket = ngx.socket +local ngx_var = ngx.var + +local warn_fmt = log.warn_fmt +local debug_fmt = log.debug_fmt + +local KEY_EXTRA_UUID = "UUID" +local KEY_EXTRA_LOCAL_ADDR = "LocalAddr" +local KEY_EXTRA_LOCAL_PORT = "LocalPort" +local KEY_EXTRA_REMOTE_ADDR = "RemoteAddr" +local KEY_EXTRA_REMOTE_PORT = "RemotePort" +local KEY_EXTRA_SCHEME = "Scheme" +local KEY_EXTRA_SERVER_NAME = "ServerName" +local KEY_EXTRA_PROXY_NAME = "ProxyName" +local KEY_EXTRA_REQ_BEGIN_TIME = "ReqBeginTime" +local KEY_EXTRA_HAS_RSP_IF_OK = "HasRspIfOK" +local KEY_EXTRA_HAS_RSP_IF_BLOCK = "HasRspIfBlock" + +local TAG_HEAD_WITH_MASK_FIRST = bor(consts.TAG_HEAD, consts.MASK_FIRST) +local TAG_EXTRA_WITH_MASK_LAST = bor(consts.TAG_EXTRA, consts.MASK_LAST) + +local T1K_PROTO = "Proto:2\n" +local T1K_PROTO_DATA = fmt("%s%s%s", char(consts.TAG_VERSION), utils.int_to_char_length(#T1K_PROTO), T1K_PROTO) + +local function read_request_body(opt_req_body_size) + local ok, err + local req_body, req_body_size + + local content_length = tonumber(ngx_var.http_content_length) or 0 + local transfer_encoding = ngx_var.http_transfer_encoding + if content_length == 0 and not transfer_encoding then + return true, nil, nil + end + + ngx_req.read_body() + req_body = ngx_req.get_body_data() + if req_body then + req_body_size = #req_body + if req_body_size > opt_req_body_size then + nlog(debug_fmt("request body is too long: %d bytes, cut to %d bytes", req_body_size, opt_req_body_size)) + req_body = sub(req_body, 1, opt_req_body_size) + end + + return true, nil, req_body + end + + local path = ngx_req.get_body_file() + if not path then + return true, nil, nil + end + + ok, err, req_body = file.read(path, opt_req_body_size) + if not ok then + err = fmt("failed to read temporary file %s: %s", path, err) + return ok, err, nil + end + + return true, nil, req_body +end + +local function get_remote_addr(remote_addr_var, remote_addr_idx) + local addr + if remote_addr_var then + addr = utils.get_indexed_element(ngx_var[remote_addr_var], remote_addr_idx) + end + return addr or ngx_var.remote_addr +end + +local function parse_v(v) + if type(v) == "table" then + return concat(v, ", ") + end + return tostring(v) +end + +local function build_header() + local http_version = ngx_req.http_version() + if http_version < 2.0 then + return true, nil, ngx_req.raw_header() + end + + local headers, err = ngx_req.get_headers(0, true) + if err then + err = fmt("failed to call ngx_req.get_headers: %s", err) + return nil, err, nil + end + + local buf = buffer:new() + buf:add(fmt("%s %s HTTP/%.1f\r\n", ngx_req.get_method(), ngx_var.request_uri, http_version)) + + for k, v in pairs(headers) do + buf:add(fmt("%s: %s\r\n", k, parse_v(v))) + end + buf:add("\r\n") + + return true, nil, buf +end + +local function build_body(opts) + local ok, err + local body + + local req_body_size = opts.req_body_size * 1024 + ok, err, body = read_request_body(req_body_size) + if not ok then + return ok, err, nil + end + + return true, nil, body +end + +local function build_extra(opts) + local err + + local src_ip = get_remote_addr(opts.remote_addr_var, opts.remote_addr_idx) + if not src_ip then + err = fmt("failed to get remote_addr, var: %s, idx %d", opts.remote_addr_var, opts.remote_addr_idx) + return nil, err + end + + local src_port = ngx_var.remote_port + if not src_port then + err = "failed to get ngx_var.remote_port" + return nil, err, nil + end + + local local_ip = ngx_var.server_addr + if not local_ip then + err = "failed to get ngx_var.server_addr" + return nil, err, nil + end + + local local_port = ngx_var.server_port + if not local_port then + err = "failed to get ngx_var.server_port" + return nil, err, nil + end + + local extra = buffer:new({ + KEY_EXTRA_UUID, ":", uuid.generate_v4(), "\n", + KEY_EXTRA_REMOTE_ADDR, ":", src_ip, "\n", + KEY_EXTRA_REMOTE_PORT, ":", src_port, "\n", + KEY_EXTRA_LOCAL_ADDR, ":", local_ip, "\n", + KEY_EXTRA_LOCAL_PORT, ":", local_port, "\n", + KEY_EXTRA_SCHEME, ":", ngx_var.scheme, "\n", + KEY_EXTRA_SERVER_NAME, ":", ngx_var.server_name, "\n", + KEY_EXTRA_PROXY_NAME, ":", ngx_var.hostname, "\n", + KEY_EXTRA_REQ_BEGIN_TIME, ":", fmt("%.0f", ngx_req.start_time() * 1000000), "\n", + KEY_EXTRA_HAS_RSP_IF_OK, ":n\n", + KEY_EXTRA_HAS_RSP_IF_BLOCK, ":n\n" + }) + + return true, nil, extra +end + +local function do_send(sock, data) + local ok, err = sock:send(data) + if not ok then + return ok, err + end + return true, nil +end + +local function receive_data(s, srv) + local t = {} + local ft = true + local finished + + repeat + local err + local tag, length, packet, rsp_body + + packet, err = s:receive(consts.T1K_HEADER_SIZE) + if err then + err = fmt("failed to receive info packet from t1k server %s: %s", srv, err) + return nil, err, nil + end + if not packet then + err = fmt("empty packet from t1k server %s", srv) + return nil, err, nil + end + + if ft then + if not utils.is_mask_first(byte(packet, 1, 1)) then + err = fmt("first packet is not MASK_FIRST from t1k server %s", srv) + return nil, err, nil + end + ft = false + end + + finished, tag, length = utils.packet_parser(packet) + if length > 0 then + rsp_body, err = s:receive(length) + if not rsp_body or #rsp_body ~= length then + err = fmt("failed to receive payload from t1k server %s: %s", srv, err) + return nil, err, nil + end + t[tag] = rsp_body + end + + until (finished) + + return true, nil, t +end + +local function get_socket(opts) + local ok, err + local count, sock, server + + sock, err = ngx_socket.tcp() + if not sock then + err = fmt("failed to create socket: %s", err) + return nil, err, nil, nil + end + + sock:settimeouts(opts.connect_timeout, opts.send_timeout, opts.read_timeout) + + if opts.uds then + server = opts.host + ok, err = sock:connect(opts.host) + else + server = fmt("%s:%d", opts.host, opts.port) + ok, err = sock:connect(opts.host, opts.port) + end + if not ok then + sock:close() + err = fmt("failed to connect to t1k server %s: %s", server, err) + return ok, err, nil, nil + end + nlog(debug_fmt("successfully connected to t1k server %s", server)) + + count, err = sock:getreusedtimes() + if not count then + nlog(warn_fmt("failed to get reused times from t1k server %s: %s", server, err)) + end + + if count == 0 then + ok, err = sock:setoption("keepalive", true) + if not ok then + nlog(warn_fmt("failed to set keepalive for t1k server %s: %s", server, err)) + end + ok, err = sock:setoption("reuseaddr", true) + if not ok then + nlog(warn_fmt("failed to set reuseaddr for t1k server %s: %s", server, err)) + end + ok, err = sock:setoption("tcp-nodelay", true) + if not ok then + nlog(warn_fmt("failed to set tcp-nodelay for t1k server %s: %s", server, err)) + end + end + + return true, nil, sock, server +end + +local function do_socket(opts, header, body, extra) + local ok, err + local t, sock, server + + ok, err, sock, server = get_socket(opts) + if not ok then + err = fmt("failed to get socket: %s", err) + return ok, err, nil + end + + ok, err = do_send(sock, { char(TAG_HEAD_WITH_MASK_FIRST), utils.int_to_char_length(header:len()), header }) + if not ok then + sock:close() + err = fmt("failed to send header data to t1k server %s: %s", server, err) + return ok, err, nil + end + + if body ~= nil then + ok, err = do_send(sock, { char(consts.TAG_BODY), utils.int_to_char_length(body:len()), body }) + if not ok then + sock:close() + err = fmt("failed to send body data to t1k server %s: %s", server, err) + return ok, err, nil + end + end + + ok, err = do_send(sock, { T1K_PROTO_DATA, char(TAG_EXTRA_WITH_MASK_LAST), utils.int_to_char_length(extra:len()), extra }) + if not ok then + sock:close() + err = fmt("failed to send extra data to t1k server %s: %s", server, err) + return ok, err, nil + end + + ok, err, t = receive_data(sock, server) + if not ok then + return ok, err, nil + end + + ok, err = sock:setkeepalive(opts.keepalive_timeout, opts.keepalive_size) + if not ok then + nlog(warn_fmt("failed to set keepalive: %s", err)) + sock:close() + end + + return true, nil, t +end + +function _M.do_request(opts) + local ok, err + local header, body, extra, t + + ok, err, header = build_header(opts) + if not ok then + return ok, err, nil + end + + ok, err, body = build_body(opts) + if not ok then + return ok, err, nil + end + + ok, err, extra = build_extra(opts) + if not ok then + return ok, err, nil + end + + ok, err, t = do_socket(opts, header, body, extra) + if not ok then + return ok, err, nil + end + + if opts.mode == consts.MODE_BLOCK then + local extra_header = t[consts.TAG_EXTRA_HEADER] + if extra_header then + ngx.ctx.t1k_extra_header = extra_header + end + end + + local result = { + action = t[consts.TAG_HEAD], + status = t[consts.TAG_BODY], + event_id = utils.get_event_id(t[consts.TAG_EXTRA_BODY]), + } + + return true, nil, result +end + +return _M diff --git a/sdk/lua-resty-t1k/lib/resty/t1k/utils.lua b/sdk/lua-resty-t1k/lib/resty/t1k/utils.lua new file mode 100644 index 0000000..b0480a1 --- /dev/null +++ b/sdk/lua-resty-t1k/lib/resty/t1k/utils.lua @@ -0,0 +1,123 @@ +local consts = require "resty.t1k.constants" + +local _M = { + _VERSION = '1.0.0' +} + +local band = bit.band +local bnot = bit.bnot +local lshift = bit.lshift +local rshift = bit.rshift + +local abs = math.abs +local char = string.char + +local ngx = ngx +local ngx_re = ngx.re +local re_match = ngx_re.match +local re_gmatch = ngx_re.gmatch +local re_gsub = ngx_re.gsub + +local NOT_MASK_FIRST = bnot(consts.MASK_FIRST) +local NOT_MASK_LAST = bnot(consts.MASK_LAST) + +function _M.int_to_char_length(x) + return char(band(x, 0xff)) .. char(band(rshift(x, 8), 0xff)) .. + char(band(rshift(x, 16), 0xff)) .. char(band(rshift(x, 24), 0xff)) +end + +function _M.char_to_int_length(l) + return l:byte(1, 1) + lshift(l:byte(2, 2), 8) + lshift(l:byte(3, 3), 16) + lshift(l:byte(4, 4), 24) +end + +function _M.is_mask_first(b) + return band(b, consts.MASK_FIRST) == consts.MASK_FIRST +end + +function _M.is_mask_last(b) + return band(b, consts.MASK_LAST) == consts.MASK_LAST +end + +local function tag_parser(tag) + return band(band(tag, NOT_MASK_FIRST), NOT_MASK_LAST) +end + +function _M.packet_parser(header) + if #header ~= consts.T1K_HEADER_SIZE then + return true, nil, 0 + end + + local fb = header:byte(1, 1) + local finished = _M.is_mask_last(fb) + local tag = tag_parser(fb) + local length = _M.char_to_int_length(header:sub(2, 5)) + + return finished, tag, length +end + +function _M.starts_with(str, start) + return str:sub(1, #start) == start +end + +function _M.to_var_idx(o) + local var = o + local idx + + local _, p = o:find(":") + if p then + var = o:sub(1, p - 1) + idx = tonumber(o:sub(p + 1)) + end + + var = re_gsub(var:lower(), "-", "_") + if not _M.starts_with(var, consts.NGX_HTTP_HEADER_PREFIX) then + var = consts.NGX_HTTP_HEADER_PREFIX .. var + end + + return var, idx +end + +function _M.get_indexed_element(str, idx) + if not str or not idx or idx == 0 then + return str + end + + local it, err = re_gmatch(str, [[([^,\s]+)]], "jo") + if err then + return nil + end + + local t = {} + for m, e in it do + if e then + return nil + end + table.insert(t, m[1]) + end + + local len = #t + if len < abs(idx) then + return nil + end + + return t[idx > 0 and idx or (len + idx + 1)] +end + +function _M.get_event_id(str) + if not str then + return nil + end + + local m, err = re_match(str, [[]], "jo") + if err then + return nil + end + + if m then + return m[1] + end + + return nil +end + +return _M diff --git a/sdk/lua-resty-t1k/lib/resty/t1k/uuid.lua b/sdk/lua-resty-t1k/lib/resty/t1k/uuid.lua new file mode 100644 index 0000000..9c04e97 --- /dev/null +++ b/sdk/lua-resty-t1k/lib/resty/t1k/uuid.lua @@ -0,0 +1,100 @@ +local bit = require "bit" +local ffi = require "ffi" + +local log = require "resty.t1k.log" + +local _M = { + _VERSION = '1.0.0' +} + +local C = ffi.C +local N_BYTES = 32 +local random = math.random + +local nlog = ngx.log + +local warn_fmt = log.warn_fmt + +ffi.cdef [[ + int RAND_bytes(unsigned char *buf, int num); +]] + +local function _rand_bytes(buf, len) + return C.RAND_bytes(buf, len) +end + +local function rand_bytes(len) + local buf = ffi.new("char[?]", len) + local ok, ret = pcall(_rand_bytes, buf, len) + + if not ok or ret ~= 1 then + nlog(warn_fmt("call RAND_bytes failed: %s", ret)) + return nil + end + + return ffi.string(buf, len) +end + +do + local band = bit.band + local bor = bit.bor + local tohex = bit.tohex + + local fmt = string.format + local byte = string.byte + + function _M.generate_v4() + local bytes = rand_bytes(N_BYTES) + + -- fallback to math.random based method + if not bytes then + return (fmt('%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s', + tohex(random(0, 255), 2), + tohex(random(0, 255), 2), + tohex(random(0, 255), 2), + tohex(random(0, 255), 2), + + tohex(random(0, 255), 2), + tohex(random(0, 255), 2), + + tohex(bor(band(random(0, 255), 0x0F), 0x40), 2), + tohex(random(0, 255), 2), + + tohex(bor(band(random(0, 255), 0x3F), 0x80), 2), + tohex(random(0, 255), 2), + + tohex(random(0, 255), 2), + tohex(random(0, 255), 2), + tohex(random(0, 255), 2), + tohex(random(0, 255), 2), + tohex(random(0, 255), 2), + tohex(random(0, 255), 2))) + end + + return fmt('%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s', + tohex(byte(bytes, 1, 2), 2), + tohex(byte(bytes, 3, 4), 2), + tohex(byte(bytes, 5, 6), 2), + tohex(byte(bytes, 7, 8), 2), + + tohex(byte(bytes, 9, 10), 2), + tohex(byte(bytes, 11, 12), 2), + + tohex(bor(band(byte(bytes, 13, 14), 0x0F), 0x40), 2), + tohex(byte(bytes, 15, 16), 2), + + tohex(bor(band(byte(bytes, 17, 18), 0x3F), 0x80), 2), + tohex(byte(bytes, 19, 20), 2), + + tohex(byte(bytes, 21, 22), 2), + tohex(byte(bytes, 23, 24), 2), + tohex(byte(bytes, 25, 26), 2), + tohex(byte(bytes, 27, 28), 2), + tohex(byte(bytes, 29, 30), 2), + tohex(byte(bytes, 31, 32), 2)) + end +end + +return setmetatable(_M, { + __call = _M.generate_v4 +}) diff --git a/sdk/lua-resty-t1k/mainspec/lua-resty-t1k-main-0-0.rockspec b/sdk/lua-resty-t1k/mainspec/lua-resty-t1k-main-0-0.rockspec new file mode 100644 index 0000000..5b9b790 --- /dev/null +++ b/sdk/lua-resty-t1k/mainspec/lua-resty-t1k-main-0-0.rockspec @@ -0,0 +1,32 @@ +package = "lua-resty-t1k-main" +version = "0-0" +source = { + url = "git://github.com/chaitin/lua-resty-t1k", + branch = "main", +} + +description = { + summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", + detailed = [[ + Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. + ]], + homepage = "https://github.com/chaitin/lua-resty-t1k", + license = "Apache License 2.0", + maintainer = "Xudong Wang " +} + +build = { + type = "builtin", + modules = { + ["resty.t1k"] = "lib/resty/t1k.lua", + ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", + ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", + ["resty.t1k.file"] = "lib/resty/t1k/file.lua", + ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", + ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", + ["resty.t1k.log"] = "lib/resty/t1k/log.lua", + ["resty.t1k.request"] = "lib/resty/t1k/request.lua", + ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", + ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", + }, +} diff --git a/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.0.0-0.rockspec b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.0.0-0.rockspec new file mode 100644 index 0000000..297b403 --- /dev/null +++ b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.0.0-0.rockspec @@ -0,0 +1,32 @@ +package = "lua-resty-t1k" +version = "1.0.0-0" +source = { + url = "git://github.com/chaitin/lua-resty-t1k", + tag = "v1.0.0" +} + +description = { + summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", + detailed = [[ + Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. + ]], + homepage = "https://github.com/chaitin/lua-resty-t1k", + license = "Apache License 2.0", + maintainer = "Xudong Wang " +} + +build = { + type = "builtin", + modules = { + ["resty.t1k"] = "lib/resty/t1k.lua", + ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", + ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", + ["resty.t1k.file"] = "lib/resty/t1k/file.lua", + ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", + ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", + ["resty.t1k.log"] = "lib/resty/t1k/log.lua", + ["resty.t1k.request"] = "lib/resty/t1k/request.lua", + ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", + ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", + }, +} diff --git a/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.0.1-0.rockspec b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.0.1-0.rockspec new file mode 100644 index 0000000..a83373a --- /dev/null +++ b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.0.1-0.rockspec @@ -0,0 +1,32 @@ +package = "lua-resty-t1k" +version = "1.0.1-0" +source = { + url = "git://github.com/chaitin/lua-resty-t1k", + tag = "v1.0.1" +} + +description = { + summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", + detailed = [[ + Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. + ]], + homepage = "https://github.com/chaitin/lua-resty-t1k", + license = "Apache License 2.0", + maintainer = "Xudong Wang " +} + +build = { + type = "builtin", + modules = { + ["resty.t1k"] = "lib/resty/t1k.lua", + ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", + ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", + ["resty.t1k.file"] = "lib/resty/t1k/file.lua", + ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", + ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", + ["resty.t1k.log"] = "lib/resty/t1k/log.lua", + ["resty.t1k.request"] = "lib/resty/t1k/request.lua", + ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", + ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", + }, +} diff --git a/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.0.2-0.rockspec b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.0.2-0.rockspec new file mode 100644 index 0000000..8b6327e --- /dev/null +++ b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.0.2-0.rockspec @@ -0,0 +1,32 @@ +package = "lua-resty-t1k" +version = "1.0.2-0" +source = { + url = "git://github.com/chaitin/lua-resty-t1k", + tag = "v1.0.2" +} + +description = { + summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", + detailed = [[ + Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. + ]], + homepage = "https://github.com/chaitin/lua-resty-t1k", + license = "Apache License 2.0", + maintainer = "Xudong Wang " +} + +build = { + type = "builtin", + modules = { + ["resty.t1k"] = "lib/resty/t1k.lua", + ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", + ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", + ["resty.t1k.file"] = "lib/resty/t1k/file.lua", + ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", + ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", + ["resty.t1k.log"] = "lib/resty/t1k/log.lua", + ["resty.t1k.request"] = "lib/resty/t1k/request.lua", + ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", + ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", + }, +} diff --git a/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.0.3-0.rockspec b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.0.3-0.rockspec new file mode 100644 index 0000000..1d264f7 --- /dev/null +++ b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.0.3-0.rockspec @@ -0,0 +1,32 @@ +package = "lua-resty-t1k" +version = "1.0.3-0" +source = { + url = "git://github.com/chaitin/lua-resty-t1k", + tag = "v1.0.3" +} + +description = { + summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", + detailed = [[ + Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. + ]], + homepage = "https://github.com/chaitin/lua-resty-t1k", + license = "Apache License 2.0", + maintainer = "Xudong Wang " +} + +build = { + type = "builtin", + modules = { + ["resty.t1k"] = "lib/resty/t1k.lua", + ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", + ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", + ["resty.t1k.file"] = "lib/resty/t1k/file.lua", + ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", + ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", + ["resty.t1k.log"] = "lib/resty/t1k/log.lua", + ["resty.t1k.request"] = "lib/resty/t1k/request.lua", + ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", + ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", + }, +} diff --git a/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.0-0.rockspec b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.0-0.rockspec new file mode 100644 index 0000000..240caa4 --- /dev/null +++ b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.0-0.rockspec @@ -0,0 +1,32 @@ +package = "lua-resty-t1k" +version = "1.1.0-0" +source = { + url = "git://github.com/chaitin/lua-resty-t1k", + tag = "v1.1.0" +} + +description = { + summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", + detailed = [[ + Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. + ]], + homepage = "https://github.com/chaitin/lua-resty-t1k", + license = "Apache License 2.0", + maintainer = "Xudong Wang " +} + +build = { + type = "builtin", + modules = { + ["resty.t1k"] = "lib/resty/t1k.lua", + ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", + ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", + ["resty.t1k.file"] = "lib/resty/t1k/file.lua", + ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", + ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", + ["resty.t1k.log"] = "lib/resty/t1k/log.lua", + ["resty.t1k.request"] = "lib/resty/t1k/request.lua", + ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", + ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", + }, +} diff --git a/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.1-0.rockspec b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.1-0.rockspec new file mode 100644 index 0000000..bea43b6 --- /dev/null +++ b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.1-0.rockspec @@ -0,0 +1,32 @@ +package = "lua-resty-t1k" +version = "1.1.1-0" +source = { + url = "git://github.com/chaitin/lua-resty-t1k", + tag = "v1.1.1" +} + +description = { + summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", + detailed = [[ + Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. + ]], + homepage = "https://github.com/chaitin/lua-resty-t1k", + license = "Apache License 2.0", + maintainer = "Xudong Wang " +} + +build = { + type = "builtin", + modules = { + ["resty.t1k"] = "lib/resty/t1k.lua", + ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", + ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", + ["resty.t1k.file"] = "lib/resty/t1k/file.lua", + ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", + ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", + ["resty.t1k.log"] = "lib/resty/t1k/log.lua", + ["resty.t1k.request"] = "lib/resty/t1k/request.lua", + ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", + ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", + }, +} diff --git a/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.2-0.rockspec b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.2-0.rockspec new file mode 100644 index 0000000..3a3fe5c --- /dev/null +++ b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.2-0.rockspec @@ -0,0 +1,32 @@ +package = "lua-resty-t1k" +version = "1.1.2-0" +source = { + url = "git://github.com/chaitin/lua-resty-t1k", + tag = "v1.1.2" +} + +description = { + summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", + detailed = [[ + Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. + ]], + homepage = "https://github.com/chaitin/lua-resty-t1k", + license = "Apache License 2.0", + maintainer = "Xudong Wang " +} + +build = { + type = "builtin", + modules = { + ["resty.t1k"] = "lib/resty/t1k.lua", + ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", + ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", + ["resty.t1k.file"] = "lib/resty/t1k/file.lua", + ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", + ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", + ["resty.t1k.log"] = "lib/resty/t1k/log.lua", + ["resty.t1k.request"] = "lib/resty/t1k/request.lua", + ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", + ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", + }, +} diff --git a/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.3-0.rockspec b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.3-0.rockspec new file mode 100644 index 0000000..acac469 --- /dev/null +++ b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.3-0.rockspec @@ -0,0 +1,32 @@ +package = "lua-resty-t1k" +version = "1.1.3-0" +source = { + url = "git://github.com/chaitin/lua-resty-t1k", + tag = "v1.1.3" +} + +description = { + summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", + detailed = [[ + Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. + ]], + homepage = "https://github.com/chaitin/lua-resty-t1k", + license = "Apache License 2.0", + maintainer = "Xudong Wang " +} + +build = { + type = "builtin", + modules = { + ["resty.t1k"] = "lib/resty/t1k.lua", + ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", + ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", + ["resty.t1k.file"] = "lib/resty/t1k/file.lua", + ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", + ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", + ["resty.t1k.log"] = "lib/resty/t1k/log.lua", + ["resty.t1k.request"] = "lib/resty/t1k/request.lua", + ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", + ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", + }, +} diff --git a/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.4-0.rockspec b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.4-0.rockspec new file mode 100644 index 0000000..73899fe --- /dev/null +++ b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.4-0.rockspec @@ -0,0 +1,32 @@ +package = "lua-resty-t1k" +version = "1.1.4-0" +source = { + url = "git://github.com/chaitin/lua-resty-t1k", + tag = "v1.1.4" +} + +description = { + summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", + detailed = [[ + Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. + ]], + homepage = "https://github.com/chaitin/lua-resty-t1k", + license = "Apache License 2.0", + maintainer = "Xudong Wang " +} + +build = { + type = "builtin", + modules = { + ["resty.t1k"] = "lib/resty/t1k.lua", + ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", + ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", + ["resty.t1k.file"] = "lib/resty/t1k/file.lua", + ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", + ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", + ["resty.t1k.log"] = "lib/resty/t1k/log.lua", + ["resty.t1k.request"] = "lib/resty/t1k/request.lua", + ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", + ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", + }, +} diff --git a/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.5-0.rockspec b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.5-0.rockspec new file mode 100644 index 0000000..f94fd78 --- /dev/null +++ b/sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.5-0.rockspec @@ -0,0 +1,32 @@ +package = "lua-resty-t1k" +version = "1.1.5-0" +source = { + url = "git://github.com/chaitin/lua-resty-t1k", + tag = "v1.1.5" +} + +description = { + summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", + detailed = [[ + Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. + ]], + homepage = "https://github.com/chaitin/lua-resty-t1k", + license = "Apache License 2.0", + maintainer = "Xudong Wang " +} + +build = { + type = "builtin", + modules = { + ["resty.t1k"] = "lib/resty/t1k.lua", + ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", + ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", + ["resty.t1k.file"] = "lib/resty/t1k/file.lua", + ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", + ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", + ["resty.t1k.log"] = "lib/resty/t1k/log.lua", + ["resty.t1k.request"] = "lib/resty/t1k/request.lua", + ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", + ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", + }, +} diff --git a/sdk/lua-resty-t1k/t/buffer.t b/sdk/lua-resty-t1k/t/buffer.t new file mode 100644 index 0000000..a08a17a --- /dev/null +++ b/sdk/lua-resty-t1k/t/buffer.t @@ -0,0 +1,36 @@ +use Test::Nginx::Socket; + +our $HttpConfig = <<'_EOC_'; + lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; +_EOC_ + +repeat_each(3); + +plan tests => repeat_each() * (blocks() * 3); + +run_tests(); + +__DATA__ + +=== TEST 1: buffer add +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local buffer = require "resty.t1k.buffer" + local b = buffer:new() + b:add("hello") + b:add(" ") + b:add("world") + b:add("!") + ngx.say(b[1], b[2], b[3], b[4]) + ngx.say(b:len()) + } + } +--- request +GET /t +--- response_body +hello world! +12 +--- no_error_log +[error] diff --git a/sdk/lua-resty-t1k/t/file.t b/sdk/lua-resty-t1k/t/file.t new file mode 100644 index 0000000..c6b8275 --- /dev/null +++ b/sdk/lua-resty-t1k/t/file.t @@ -0,0 +1,136 @@ +use Test::Nginx::Socket; + +our $HttpConfig = <<'_EOC_'; + lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; +_EOC_ + +repeat_each(3); + +plan tests => repeat_each() * (blocks() * 3); + +run_tests(); + +__DATA__ + +=== TEST 1: read full file +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local file = require "resty.t1k.file" + local path = ngx.var.document_root .. "/foo.bar" + local ok, err, content = file.read(path, 2 ^ 15) + if not ok then + ngx.say(err) + end + ngx.print(table.concat(content)) + } + } +--- user_files eval +[ + ["foo.bar" => "a" x (2 ** 14) ], +] +--- request +GET /t +--- response_body eval +"a" x (2 ** 14) +--- no_error_log +[error] + + + +=== TEST 2: read partial file +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local file = require "resty.t1k.file" + local path = ngx.var.document_root .. "/foo.bar" + local ok, err, content = file.read(path, 2 ^ 13) + if not ok then + ngx.say(err) + end + ngx.print(table.concat(content)) + } + } +--- user_files eval +[ + ["foo.bar" => "a" x (2 ** 14) ], +] +--- request +GET /t +--- response_body eval +"a" x (2 ** 13) +--- no_error_log +[error] + + + +=== TEST 3: read negative bytes +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local file = require "resty.t1k.file" + local path = ngx.var.document_root .. "/foo.bar" + local ok, err, content = file.read(path, -1) + if not ok then + ngx.say(err) + end + ngx.print(table.concat(content)) + } + } +--- user_files +>>> foo.bar +--- request +GET /t +--- response_body eval +"" +--- no_error_log +[error] + + + +=== TEST 4: read empty file +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local file = require "resty.t1k.file" + local path = ngx.var.document_root .. "/foo.bar" + local ok, err, content = file.read(path, 1) + if not ok then + ngx.say(err) + end + ngx.print(table.concat(content)) + } + } +--- user_files +>>> foo.bar +--- request +GET /t +--- response_body eval +"" +--- no_error_log +[error] + + + +=== TEST 5: read non-existent file +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local file = require "resty.t1k.file" + local ok, err, buffer = file.read("/opt/non_existent_file", 0) + if not ok then + ngx.say(err) + end + } + } +--- request +GET /t +--- response_body +/opt/non_existent_file: No such file or directory +--- no_error_log +[error] diff --git a/sdk/lua-resty-t1k/t/filter.t b/sdk/lua-resty-t1k/t/filter.t new file mode 100644 index 0000000..8098cc6 --- /dev/null +++ b/sdk/lua-resty-t1k/t/filter.t @@ -0,0 +1,39 @@ +use Test::Nginx::Socket; + +our $HttpConfig = <<'_EOC_'; + lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; +_EOC_ + +repeat_each(3); + +plan tests => repeat_each() * (blocks() * 5); + +run_tests(); + +__DATA__ + +=== TEST 1: do_header_filter +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + ngx.ctx.t1k_extra_header = "k1:v1\nk2:v2\nk3:v3\n" + } + + header_filter_by_lua_block { + local filter = require "resty.t1k.filter" + filter.do_header_filter() + } + + content_by_lua_block { + ngx.say("hi") + } + } +--- request +GET /t +--- response_headers +k1: v1 +k2: v2 +k3: v3 +--- no_error_log +[error] diff --git a/sdk/lua-resty-t1k/t/handler.t b/sdk/lua-resty-t1k/t/handler.t new file mode 100644 index 0000000..df83223 --- /dev/null +++ b/sdk/lua-resty-t1k/t/handler.t @@ -0,0 +1,138 @@ +use Test::Nginx::Socket; + +our $HttpConfig = <<'_EOC_'; + lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; +_EOC_ + +repeat_each(3); + +plan tests => repeat_each() * (blocks() * 3); + +run_tests(); + +__DATA__ + +=== TEST 1: handle passed action +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local handler = require "resty.t1k.handler" + + local t = { + action = ".", + } + + local ok, err = handler.handle(t) + if not ok then + ngx.log(ngx.ERR, err) + end + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 2: handle blocked action +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local handler = require "resty.t1k.handler" + + local t = { + action = "?", + status = 405, + event_id = "c0c039a7c348486eaffd9e2f9846b66b", + } + + local ok, err = handler.handle(t) + if not ok then + ngx.log(ngx.ERR, err) + end + } + + header_filter_by_lua_block { + local filter = require "resty.t1k.filter" + filter.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- request +GET /t +--- response_body +{"code": 405, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "c0c039a7c348486eaffd9e2f9846b66b"} +--- error_code eval +"405" +--- no_error_log +[error] +--- no_error_log +[error] + + + +=== TEST 3: handle unknown action +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local handler = require "resty.t1k.handler" + + local t = { + action = "~" + } + + local ok, err = handler.handle(t) + if not ok then + ngx.log(ngx.ERR, err) + end + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- request +GET /t +--- response_body +passed +--- error_log +lua-resty-t1k: unknown action from t1k server: ~ + + + +=== TEST 4: handle nil result +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local handler = require "resty.t1k.handler" + + local ok, err = handler.handle(nil) + if not ok then + ngx.log(ngx.ERR, err) + end + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- request +GET /t +--- response_body +passed +--- error_log +lua-resty-t1k: invalid result type: nil diff --git a/sdk/lua-resty-t1k/t/integration.t b/sdk/lua-resty-t1k/t/integration.t new file mode 100644 index 0000000..77110ca --- /dev/null +++ b/sdk/lua-resty-t1k/t/integration.t @@ -0,0 +1,1142 @@ +use Test::Nginx::Socket 'no_plan'; + +our $MainConfig = <<'_EOC_'; +env DETECTOR_IP; +_EOC_ + +our $HttpConfig = <<'_EOC_'; + lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; + lua_socket_log_errors off; +_EOC_ + +run_tests(); + +__DATA__ + +=== TEST 1: integration test blocked +--- main_config eval: $::MainConfig +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "block", + host = os.getenv("DETECTOR_IP"), + port = 8000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t) + if not ok then + ngx.log(ngx.ERR, err) + return + end + + if t.mode ~= "block" then + ngx.log(ngx.DEBUG, "skip blocking") + return + end + + ok, err = t1k.do_handle(result) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- request +GET /t/shell.php +--- response_headers +Content-Type: application/json +--- response_body_like eval +'^{"code": 403, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": ".*"}$' +--- error_code: 403 +--- no_error_log +[error] +--- error_log eval +"lua-resty-t1k: successfully connected to t1k server $ENV{DETECTOR_IP}:8000" +--- log_level: debug +--- skip_eval +4: not exists($ENV{DETECTOR_IP}) + + + +=== TEST 2: integration test blocked internal handle +--- main_config eval: $::MainConfig +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "block", + host = os.getenv("DETECTOR_IP"), + port = 8000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t, true) + if not ok then + ngx.log(ngx.ERR, err) + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- request +GET /t/shell.php +--- response_headers +Content-Type: application/json +--- response_body_like eval +'^{"code": 403, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": ".*"}$' +--- error_code: 403 +--- no_error_log +[error] +--- error_log eval +"lua-resty-t1k: successfully connected to t1k server $ENV{DETECTOR_IP}:8000" +--- log_level: debug +--- skip_eval +4: not exists($ENV{DETECTOR_IP}) + + + +=== TEST 3: integration test blocked http2 +--- main_config eval: $::MainConfig +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "block", + host = os.getenv("DETECTOR_IP"), + port = 8000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t) + if not ok then + ngx.log(ngx.ERR, err) + return + end + + if t.mode ~= "block" then + ngx.log(ngx.DEBUG, "skip blocking") + return + end + + ok, err = t1k.do_handle(result) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- http2 +--- request +GET /t/shell.php +--- response_headers +Content-Type: application/json +--- response_body_like eval +'^{"code": 403, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": ".*"}$' +--- error_code: 403 +--- no_error_log +[error] +--- error_log eval +"lua-resty-t1k: successfully connected to t1k server $ENV{DETECTOR_IP}:8000" +--- log_level: debug +--- skip_eval +4: not exists($ENV{DETECTOR_IP}) + + + +=== TEST 4: integration test blocked http2 internal handle +--- main_config eval: $::MainConfig +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "block", + host = os.getenv("DETECTOR_IP"), + port = 8000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t, true) + if not ok then + ngx.log(ngx.ERR, err) + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- http2 +--- request +GET /t/shell.php +--- response_headers +Content-Type: application/json +--- response_body_like eval +'^{"code": 403, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": ".*"}$' +--- error_code: 403 +--- no_error_log +[error] +--- error_log eval +"lua-resty-t1k: successfully connected to t1k server $ENV{DETECTOR_IP}:8000" +--- log_level: debug +--- skip_eval +4: not exists($ENV{DETECTOR_IP}) + + + +=== TEST 5: integration test monitor +--- main_config eval: $::MainConfig +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "monitor", + host = os.getenv("DETECTOR_IP"), + port = 8000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t) + if not ok then + ngx.log(ngx.ERR, err) + return + end + + if t.mode ~= "block" then + ngx.log(ngx.DEBUG, "skip blocking") + return + end + + ok, err = t1k.do_handle(result) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- request +GET /t/shell.php +--- response_body +passed +--- no_error_log +[error] +--- error_log eval +"lua-resty-t1k: successfully connected to t1k server $ENV{DETECTOR_IP}:8000" +--- log_level: debug +--- skip_eval +4: not exists($ENV{DETECTOR_IP}) + + + +=== TEST 6: integration test monitor internal handle +--- main_config eval: $::MainConfig +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "monitor", + host = os.getenv("DETECTOR_IP"), + port = 8000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t, true) + if not ok then + ngx.log(ngx.ERR, err) + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- request +GET /t/shell.php +--- response_body +passed +--- no_error_log +[error] +--- error_log eval +"lua-resty-t1k: successfully connected to t1k server $ENV{DETECTOR_IP}:8000" +--- log_level: debug +--- skip_eval +4: not exists($ENV{DETECTOR_IP}) + + + +=== TEST 7: integration test monitor http2 +--- main_config eval: $::MainConfig +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "monitor", + host = os.getenv("DETECTOR_IP"), + port = 8000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t) + if not ok then + ngx.log(ngx.ERR, err) + return + end + + if t.mode ~= "block" then + ngx.log(ngx.DEBUG, "skip blocking") + return + end + + ok, err = t1k.do_handle(result) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- http2 +--- request +GET /t/shell.php +--- response_body +passed +--- no_error_log +[error] +--- error_log eval +["lua-resty-t1k: successfully connected to t1k server $ENV{DETECTOR_IP}:8000", "skip blocking"] +--- log_level: debug +--- skip_eval +4: not exists($ENV{DETECTOR_IP}) + + + +=== TEST 8: integration test monitor http2 internal handle +--- main_config eval: $::MainConfig +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "monitor", + host = os.getenv("DETECTOR_IP"), + port = 8000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t, true) + if not ok then + ngx.log(ngx.ERR, err) + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- http2 +--- request +GET /t/shell.php +--- response_body +passed +--- no_error_log +[error] +--- error_log eval +"lua-resty-t1k: successfully connected to t1k server $ENV{DETECTOR_IP}:8000" +--- log_level: debug +--- skip_eval +4: not exists($ENV{DETECTOR_IP}) + + + +=== TEST 9: integration test disabled +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "off", + } + + local ok, err, result = t1k.do_access(t) + if not ok then + ngx.log(ngx.ERR, err) + return + end + + if t.mode ~= "block" then + ngx.log(ngx.DEBUG, "skip blocking") + return + end + + ok, err = t1k.do_handle(result) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- request +GET /t/shell.php +--- response_body +passed +--- no_error_log +[error] +--- error_log +lua-resty-t1k: t1k is not enabled +skip blocking +--- log_level: debug + + + +=== TEST 10: integration test configuration priority +--- main_config eval: $::MainConfig +--- http_config eval: $::HttpConfig +--- config + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "block", + host = os.getenv("DETECTOR_IP"), + port = 8000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t) + if not ok then + ngx.log(ngx.ERR, err) + return + end + + if t.mode ~= "block" then + ngx.log(ngx.DEBUG, "skip blocking") + return + end + + ok, err = t1k.do_handle(result) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + location /pass { + access_by_lua_block { + } + + content_by_lua_block { + ngx.say("passed") + } + } + + location /block { + content_by_lua_block { + ngx.say("there must be a problem when you see this line") + } + } +--- request eval +["GET /pass/shell.php", "GET /block/shell.php"] +--- response_body_like eval +["passed", '^{"code": 403, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": ".*"}$'] +--- error_code eval +[200, 403] +--- no_error_log +[error] +--- skip_eval +6: not exists($ENV{DETECTOR_IP}) + + + +=== TEST 11: integration test blocked extra headers +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "block", + host = "127.0.0.1", + port = 18000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t) + if not ok then + ngx.log(ngx.ERR, err) + return + end + + if t.mode ~= "block" then + ngx.log(ngx.DEBUG, "skip blocking") + return + end + + ok, err = t1k.do_handle(result) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- tcp_listen: 18000 +--- tcp_reply eval +"\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\x23\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a\xa4\x33\x00\x00\x00" +--- request +GET /t/shell.php +--- response_headers +k1: v1 +k2: v2 +k3: v3 +--- response_headers +Content-Type: application/json +--- response_body +{"code": 405, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "c0c039a7c348486eaffd9e2f9846b66b"} +--- error_code eval +"405" +--- no_error_log +[error] +--- error_log +lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 +--- log_level: debug + + + +=== TEST 12: integration test blocked extra headers internal handle +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "block", + host = "127.0.0.1", + port = 18000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t, true) + if not ok then + ngx.log(ngx.ERR, err) + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- tcp_listen: 18000 +--- tcp_reply eval +"\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\x23\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a\xa4\x33\x00\x00\x00" +--- request +GET /t/shell.php +--- response_headers +k1: v1 +k2: v2 +k3: v3 +--- response_headers +Content-Type: application/json +--- response_body +{"code": 405, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "c0c039a7c348486eaffd9e2f9846b66b"} +--- error_code eval +"405" +--- no_error_log +[error] +--- error_log +lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 +--- log_level: debug + + + +=== TEST 13: integration test blocked extra headers with unix domain socket +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "block", + host = "unix:t1k.sock", + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t) + if not ok then + ngx.log(ngx.ERR, err) + return + end + + if t.mode ~= "block" then + ngx.log(ngx.DEBUG, "skip blocking") + return + end + + ok, err = t1k.do_handle(result) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- tcp_listen: t1k.sock +--- tcp_reply eval +"\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\x23\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a\xa4\x33\x00\x00\x00" +--- request +GET /t/shell.php +--- response_headers +k1: v1 +k2: v2 +k3: v3 +--- response_headers +Content-Type: application/json +--- response_body +{"code": 405, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "c0c039a7c348486eaffd9e2f9846b66b"} +--- error_code eval +"405" +--- no_error_log +[error] +--- error_log +lua-resty-t1k: successfully connected to t1k server unix:t1k.sock +--- log_level: debug + + + +=== TEST 14: integration test passed extra headers +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "block", + host = "127.0.0.1", + port = 18000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t) + if not ok then + ngx.log(ngx.ERR, err) + return + end + + if t.mode ~= "block" then + ngx.log(ngx.DEBUG, "skip blocking") + return + end + + ok, err = t1k.do_handle(result) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- tcp_listen: 18000 +--- tcp_reply eval +"\x41\x01\x00\x00\x00.\xa3\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a" +--- request +GET /t +--- response_headers +k1: v1 +k2: v2 +k3: v3 +--- response_body +passed +--- no_error_log +[error] +--- error_log +lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 +--- log_level: debug + + + +=== TEST 15: integration test passed extra headers internal handle +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "block", + host = "127.0.0.1", + port = 18000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t, true) + if not ok then + ngx.log(ngx.ERR, err) + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- tcp_listen: 18000 +--- tcp_reply eval +"\x41\x01\x00\x00\x00.\xa3\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a" +--- request +GET /t +--- response_headers +k1: v1 +k2: v2 +k3: v3 +--- response_body +passed +--- no_error_log +[error] +--- error_log +lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 +--- log_level: debug + + + +=== TEST 16: integration test passed extra headers with unix domain socket +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "block", + host = "unix:t1k.sock", + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t) + if not ok then + ngx.log(ngx.ERR, err) + return + end + + if t.mode ~= "block" then + ngx.log(ngx.DEBUG, "skip blocking") + return + end + + ok, err = t1k.do_handle(result) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- tcp_listen: t1k.sock +--- tcp_reply eval +"\x41\x01\x00\x00\x00.\xa3\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a" +--- request +GET /t +--- response_headers +k1: v1 +k2: v2 +k3: v3 +--- response_body +passed +--- no_error_log +[error] +--- error_log +lua-resty-t1k: successfully connected to t1k server unix:t1k.sock +--- log_level: debug + + + +=== TEST 17: integration test monitor extra headers +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "monitor", + host = "127.0.0.1", + port = 18000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t) + if not ok then + ngx.log(ngx.ERR, err) + return + end + + if t.mode ~= "block" then + ngx.log(ngx.DEBUG, "skip blocking") + return + end + + ok, err = t1k.do_handle(result) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- tcp_listen: 18000 +--- tcp_reply eval +"\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\x23\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a\xa4\x33\x00\x00\x00" +--- request +GET /t/shell.php +--- raw_response_headers_unlike eval +'.*k1: v1\r\n.*' +--- response_body +passed +--- no_error_log +[error] +--- error_log +lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 +skip blocking +--- log_level: debug + + + +=== TEST 18: integration test monitor extra headers internal handle +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "monitor", + host = "127.0.0.1", + port = 18000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t, true) + if not ok then + ngx.log(ngx.ERR, err) + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- tcp_listen: 18000 +--- tcp_reply eval +"\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\x23\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a\xa4\x33\x00\x00\x00" +--- request +GET /t/shell.php +--- raw_response_headers_unlike eval +'.*k1: v1\r\n.*' +--- response_body +passed +--- no_error_log +[error] +--- error_log +lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 +--- log_level: debug + + + +=== TEST 19: integration test monitor extra headers with unix domain socket +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "monitor", + host = "unix:t1k.sock", + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = t1k.do_access(t) + if not ok then + ngx.log(ngx.ERR, err) + return + end + + if t.mode ~= "block" then + ngx.log(ngx.DEBUG, "skip blocking") + return + end + + ok, err = t1k.do_handle(result) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + header_filter_by_lua_block { + local t1k = require "resty.t1k" + t1k.do_header_filter() + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- tcp_listen: t1k.sock +--- tcp_reply eval +"\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\x23\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a\xa4\x33\x00\x00\x00" +--- request +GET /t/shell.php +--- raw_response_headers_unlike eval +'.*k1: v1\r\n.*' +--- response_body +passed +--- no_error_log +[error] +--- error_log +lua-resty-t1k: successfully connected to t1k server unix:t1k.sock +skip blocking +--- log_level: debug diff --git a/sdk/lua-resty-t1k/t/log.t b/sdk/lua-resty-t1k/t/log.t new file mode 100644 index 0000000..8e5853a --- /dev/null +++ b/sdk/lua-resty-t1k/t/log.t @@ -0,0 +1,67 @@ +use Test::Nginx::Socket; + +our $HttpConfig = <<'_EOC_'; + lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; +_EOC_ + +repeat_each(3); + +plan tests => repeat_each() * (blocks() * 2 + 2); + +run_tests(); + +__DATA__ + +=== TEST 1: err_fmt +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local log = require "resty.t1k.log" + ngx.log(log.err_fmt("%s - %04d - %.4f", "test", 1, 1)) + } + } +--- request +GET /t +[error] +--- error_log +lua-resty-t1k: test - 0001 - 1.0000 +--- log_level: error + + + +=== TEST 2: warn_fmt +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local log = require "resty.t1k.log" + ngx.log(log.warn_fmt("%s - %04d - %.4f", "test", 1, 1)) + } + } +--- request +GET /t +--- no_error_log +[error] +--- error_log +lua-resty-t1k: test - 0001 - 1.0000 +--- log_level: warn + + + +=== TEST 3: debug_fmt +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local log = require "resty.t1k.log" + ngx.log(log.debug_fmt("%s - %04d - %.4f", "test", 1, 1)) + } + } +--- request +GET /t +--- no_error_log +[error] +--- error_log +lua-resty-t1k: test - 0001 - 1.0000 +--- log_level: debug diff --git a/sdk/lua-resty-t1k/t/option.t b/sdk/lua-resty-t1k/t/option.t new file mode 100644 index 0000000..b10e53d --- /dev/null +++ b/sdk/lua-resty-t1k/t/option.t @@ -0,0 +1,169 @@ +use Test::Nginx::Socket; + +our $HttpConfig = <<'_EOC_'; + lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; +_EOC_ + +repeat_each(3); + +plan tests => repeat_each() * (blocks() * 3 + 1); + +run_tests(); + +__DATA__ + +=== TEST 1: do_access nil option +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local ok, err, _ = t1k.do_access(nil) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- request +GET /t/shell.php +--- response_body +passed +--- no_error_log +[debug] +--- log_level: debug + + + +=== TEST 2: do_access disabled +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "off", + } + + local ok, err, _ = t1k.do_access(t) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- request +GET /t/shell.php +--- response_body +passed +--- no_error_log +[error] +--- error_log +lua-resty-t1k: t1k is not enabled +--- log_level: debug + + + +=== TEST 3: do_access invalid mode +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "invalid", + } + + local ok, err, _ = t1k.do_access(t) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- request +GET /t/shell.php +--- response_body +passed +--- error_log +lua-resty-t1k: invalid t1k mode: invalid + + + +=== TEST 4: do_access invalid host +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "block" + } + + local ok, err, _ = t1k.do_access(t) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- request +GET /t +--- response_body +passed +--- error_log +lua-resty-t1k: invalid t1k host: nil +--- log_level: debug + + + +=== TEST 5: do_access invalid port +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local t1k = require "resty.t1k" + + local t = { + mode = "block", + host = "127.0.0.1" + } + + local ok, err, _ = t1k.do_access(t) + if not ok then + ngx.log(ngx.ERR, err) + return + end + } + + content_by_lua_block { + ngx.say("passed") + } + } +--- request +GET /t +--- response_body +passed +--- error_log +lua-resty-t1k: invalid t1k port: nil +--- log_level: debug diff --git a/sdk/lua-resty-t1k/t/request.t b/sdk/lua-resty-t1k/t/request.t new file mode 100644 index 0000000..55e8991 --- /dev/null +++ b/sdk/lua-resty-t1k/t/request.t @@ -0,0 +1,640 @@ +use Test::Nginx::Socket; + +our $HttpConfig = <<'_EOC_'; + lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; +_EOC_ + +repeat_each(3); + +plan tests => repeat_each() * (blocks() * 3 + 14); + +run_tests(); + +__DATA__ + +=== TEST 1: do_request blocked +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local request = require "resty.t1k.request" + + local t = { + mode = "block", + host = "127.0.0.1", + port = 18000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = request.do_request(t) + if not ok then + ngx.log(ngx.ERR, err) + end + + ngx.say(result["action"]) + ngx.say(result["status"]) + ngx.say(result["event_id"]) + } + } +--- tcp_listen: 18000 +--- tcp_reply eval +"\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\xa4\x33\x00\x00\x00" +--- request +GET /t/shell.php +--- response_body +? +405 +c0c039a7c348486eaffd9e2f9846b66b +--- no_error_log +[error] +--- error_log +lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 +--- log_level: debug + + + +=== TEST 2: do_request blocked with unix domain socket +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local request = require "resty.t1k.request" + + local t = { + mode = "block", + host = "unix:t1k.sock", + uds = true, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = request.do_request(t) + if not ok then + ngx.log(ngx.ERR, err) + end + + ngx.say(result["action"]) + ngx.say(result["status"]) + ngx.say(result["event_id"]) + } + } +--- tcp_listen: t1k.sock +--- tcp_reply eval +"\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\xa4\x33\x00\x00\x00" +--- request +GET /t/shell.php +--- response_body +? +405 +c0c039a7c348486eaffd9e2f9846b66b +--- no_error_log +[error] +--- error_log +lua-resty-t1k: successfully connected to t1k server unix:t1k.sock +--- log_level: debug + + + +=== TEST 3: do_request passed +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local request = require "resty.t1k.request" + + local t = { + mode = "block", + host = "127.0.0.1", + port = 18000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = request.do_request(t) + if not ok then + ngx.log(ngx.ERR, err) + end + + ngx.say(result["action"]) + } + } +--- tcp_listen: 18000 +--- tcp_reply eval +"\xc1\x01\x00\x00\x00." +--- request +GET /t +--- response_body +. +--- no_error_log +[error] +--- error_log +lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 +--- log_level: debug + + + +=== TEST 4: do_request passed with unix domain socket +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local request = require "resty.t1k.request" + + local t = { + mode = "block", + host = "unix:t1k.sock", + uds = true, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = request.do_request(t) + if not ok then + ngx.log(ngx.ERR, err) + end + + ngx.say(result["action"]) + } + } +--- tcp_listen: t1k.sock +--- tcp_reply eval +"\xc1\x01\x00\x00\x00." +--- request +GET /t +--- response_body +. +--- no_error_log +[error] +--- error_log +lua-resty-t1k: successfully connected to t1k server unix:t1k.sock +--- log_level: debug + + + +=== TEST 5: do_request trim request body +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local request = require "resty.t1k.request" + + local t = { + mode = "block", + host = "127.0.0.1", + port = 18000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 0.0625, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = request.do_request(t) + if not ok then + ngx.log(ngx.ERR, err) + end + + ngx.say(result["action"]) + } + } +--- tcp_listen: 18000 +--- tcp_reply eval +"\xc1\x01\x00\x00\x00." +--- request +GET /t +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. +--- response_body +. +--- no_error_log +[error] +--- error_log +lua-resty-t1k: request body is too long: 123 bytes, cut to 64 bytes +lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 +--- log_level: debug + + + +=== TEST 6: do_request trim request body with unix domain socket +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local request = require "resty.t1k.request" + + local t = { + mode = "block", + host = "unix:t1k.sock", + uds = true, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 0.0625, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = request.do_request(t) + if not ok then + ngx.log(ngx.ERR, err) + end + + ngx.say(result["action"]) + } + } +--- tcp_listen: t1k.sock +--- tcp_reply eval +"\xc1\x01\x00\x00\x00." +--- request +GET /t +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. +--- response_body +. +--- no_error_log +[error] +--- error_log +lua-resty-t1k: request body is too long: 123 bytes, cut to 64 bytes +lua-resty-t1k: successfully connected to t1k server unix:t1k.sock +--- log_level: debug + + + +=== TEST 7: do_request refuse connection +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local request = require "resty.t1k.request" + + local t = { + mode = "block", + host = "127.0.0.1", + port = 18000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 0.0625, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = request.do_request(t) + if not ok then + ngx.log(ngx.ERR, err) + end + + ngx.say("result: ", result) + } + } +--- request +GET /t +--- response_body +result: nil +--- error_log +failed to connect to t1k server 127.0.0.1:18000 +--- log_level: debug + + + +=== TEST 8: do_request refuse connection with unix domain socket +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local request = require "resty.t1k.request" + + local t = { + mode = "block", + host = "unix:t1k.sock", + uds = true, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 0.0625, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = request.do_request(t) + if not ok then + ngx.log(ngx.ERR, err) + end + + ngx.say("result: ", result) + } + } +--- request +GET /t +--- response_body +result: nil +--- error_log +failed to connect to t1k server unix:t1k.sock +--- log_level: debug + + + +=== TEST 9: do_request timeout +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local request = require "resty.t1k.request" + + local t = { + mode = "block", + host = "127.0.0.1", + port = 18000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 100, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = request.do_request(t) + if not ok then + ngx.log(ngx.ERR, err) + end + + ngx.say("result: ", result) + } + } +--- tcp_listen: 18000 +--- tcp_reply_delay: 200ms +--- tcp_reply eval +"\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\xa4\x33\x00\x00\x00" +--- request +GET /t/shell.php +--- response_body +result: nil +--- error_log +failed to receive info packet from t1k server 127.0.0.1:18000: timeout +--- log_level: debug + + + +=== TEST 10: do_request timeout with unix domain socket +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local request = require "resty.t1k.request" + + local t = { + mode = "block", + host = "unix:t1k.sock", + uds = true, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 100, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = request.do_request(t) + if not ok then + ngx.log(ngx.ERR, err) + end + + ngx.say("result: ", result) + } + } +--- tcp_listen: t1k.sock +--- tcp_reply_delay: 200ms +--- tcp_reply eval +"\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\xa4\x33\x00\x00\x00" +--- request +GET /t/shell.php +--- response_body +result: nil +--- error_log +failed to receive info packet from t1k server unix:t1k.sock: timeout +--- log_level: debug + + + +=== TEST 11: do_request invalid action +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local request = require "resty.t1k.request" + + local t = { + mode = "block", + host = "127.0.0.1", + port = 18000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 0.0625, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = request.do_request(t) + if not ok then + ngx.log(ngx.ERR, err) + end + + ngx.say("action: ", result["action"]) + } + } +--- tcp_listen: 18000 +--- tcp_reply eval +"\xc1\x01\x00\x00\x00~" +--- request +GET /t +--- response_body +action: ~ +--- no_error_log +[error] +--- error_log +successfully connected to t1k server 127.0.0.1:18000 +--- log_level: debug + + + +=== TEST 12: do_request invalid action with unix domain socket +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local request = require "resty.t1k.request" + + local t = { + mode = "block", + host = "unix:t1k.sock", + uds = true, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 0.0625, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = request.do_request(t) + if not ok then + ngx.log(ngx.ERR, err) + end + + ngx.say("action: ", result["action"]) + } + } +--- tcp_listen: t1k.sock +--- tcp_reply eval +"\xc1\x01\x00\x00\x00~" +--- request +GET /t +--- response_body +action: ~ +--- no_error_log +[error] +--- error_log +successfully connected to t1k server unix:t1k.sock +--- log_level: debug + + + +=== TEST 13: do_request remote address +--- http_config eval: $::HttpConfig +--- config + location /t { + access_by_lua_block { + local utils = require "resty.t1k.utils" + ngx.say("ngx.var.http_x_real_ip or ngx.var.remote_addr is ", utils.get_indexed_element(ngx.var.http_x_real_ip) or ngx.var.remote_addr) + ngx.say("ngx.var.http_x_forwarded_for: 2 or ngx.var.remote_addr is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, 2) or ngx.var.remote_addr) + ngx.say("ngx.var.http_x_forwarded_for: -2 or ngx.var.remote_addr is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, -2) or ngx.var.remote_addr) + ngx.say("ngx.var.http_non_existent_header or ngx.var.remote_addr is ", utils.get_indexed_element(ngx.var.http_non_existent_header) or ngx.var.remote_addr) + } + } +--- request +GET /t +--- more_headers +X-Forwarded-For: 1.1.1.1, 2.2.2.2, 2001:db8:3333:4444:5555:6666:7777:8888, 3.3.3.3 +X-Real-IP: 100.100.100.100 +--- response_body +ngx.var.http_x_real_ip or ngx.var.remote_addr is 100.100.100.100 +ngx.var.http_x_forwarded_for: 2 or ngx.var.remote_addr is 2.2.2.2 +ngx.var.http_x_forwarded_for: -2 or ngx.var.remote_addr is 2001:db8:3333:4444:5555:6666:7777:8888 +ngx.var.http_non_existent_header or ngx.var.remote_addr is 127.0.0.1 +--- no_error_log +[error] + + + +=== TEST 14: do_request http2 +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local request = require "resty.t1k.request" + + local t = { + mode = "block", + host = "127.0.0.1", + port = 18000, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = request.do_request(t) + if not ok then + ngx.log(ngx.ERR, err) + end + + ngx.say(result["action"]) + ngx.say(result["status"]) + ngx.say(result["event_id"]) + } + } +--- tcp_listen: 18000 +--- tcp_reply eval +"\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\xa4\x33\x00\x00\x00" +--- http2 +--- request +GET /t/shell.php +--- tcp_query eval +qr/.*HTTP\/2.0.*/ +--- response_body +? +405 +c0c039a7c348486eaffd9e2f9846b66b +--- no_error_log +[error] +--- error_log +lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 +--- log_level: debug + + + +=== TEST 15: do_request http2 with unix domain socket +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local request = require "resty.t1k.request" + + local t = { + mode = "block", + host = "unix:t1k.sock", + uds = true, + connect_timeout = 1000, + send_timeout = 1000, + read_timeout = 1000, + req_body_size = 1024, + keepalive_size = 16, + keepalive_timeout = 10000, + } + + local ok, err, result = request.do_request(t) + if not ok then + ngx.log(ngx.ERR, err) + end + + ngx.say(result["action"]) + ngx.say(result["status"]) + ngx.say(result["event_id"]) + } + } +--- tcp_listen: t1k.sock +--- tcp_reply eval +"\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\xa4\x33\x00\x00\x00" +--- http2 +--- request +GET /t/shell.php +--- tcp_query eval +qr/.*HTTP\/2.0.*/ +--- response_body +? +405 +c0c039a7c348486eaffd9e2f9846b66b +--- no_error_log +[error] +--- error_log +lua-resty-t1k: successfully connected to t1k server unix:t1k.sock +--- log_level: debug diff --git a/sdk/lua-resty-t1k/t/utils.t b/sdk/lua-resty-t1k/t/utils.t new file mode 100644 index 0000000..50fb3f3 --- /dev/null +++ b/sdk/lua-resty-t1k/t/utils.t @@ -0,0 +1,264 @@ +use Test::Nginx::Socket; + +our $HttpConfig = <<'_EOC_'; + lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; +_EOC_ + +repeat_each(3); + +plan tests => repeat_each() * (blocks() * 3); + +run_tests(); + +__DATA__ + +=== TEST 1: int_to_char_length +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local utils = require "resty.t1k.utils" + ngx.say("255 to char length: ", utils.int_to_char_length(255)) + ngx.print("16777216 to char length: ", utils.int_to_char_length(16777216)) + } + } +--- request +GET /t +--- response_body eval +"255 to char length: \x{ff}\x{00}\x{00}\x{00} +16777216 to char length: \x{00}\x{00}\x{00}\x{01}" +--- no_error_log +[error] + + + +=== TEST 2: int_to_char_length +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local utils = require "resty.t1k.utils" + ngx.say("0xff 0x00 0x00 0x00 to int length: ", utils.char_to_int_length("\xff\x00\x00\x00")) + ngx.say("0x00 0x00 0x00 0x01 to int length: ", utils.char_to_int_length("\x00\x00\x00\x01")) + } + } +--- request +GET /t +--- response_body +0xff 0x00 0x00 0x00 to int length: 255 +0x00 0x00 0x00 0x01 to int length: 16777216 +--- no_error_log +[error] + + + +=== TEST 3: is_mask_first +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local utils = require "resty.t1k.utils" + ngx.say("0x30 is MASK FIRST: ", utils.is_mask_first(0x30)) + ngx.say("0x41 is MASK FIRST: ", utils.is_mask_first(0x41)) + ngx.say("0xc0 is MASK FIRST: ", utils.is_mask_first(0xc0)) + ngx.say("0xc1 is MASK FIRST: ", utils.is_mask_first(0xc1)) + } + } +--- request +GET /t +--- response_body +0x30 is MASK FIRST: false +0x41 is MASK FIRST: true +0xc0 is MASK FIRST: true +0xc1 is MASK FIRST: true +--- no_error_log +[error] + + + +=== TEST 4: is_mask_last +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local utils = require "resty.t1k.utils" + ngx.say("0x41 is MASK LAST: ", utils.is_mask_last(65)) + ngx.say("0x80 is MASK LAST: ", utils.is_mask_last(128)) + } + } +--- request +GET /t +--- response_body +0x41 is MASK LAST: false +0x80 is MASK LAST: true +--- no_error_log +[error] + + + +=== TEST 5: packet_parser unfinished +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local utils = require "resty.t1k.utils" + local finished, tag, length = utils.packet_parser("\x41\x59\x00\x00\x00") + ngx.say("finished: ", finished) + ngx.say("tag == TAG_HEAD: ", tag == 1) + ngx.say("length: ", 89) + } + } +--- request +GET /t +--- response_body +finished: false +tag == TAG_HEAD: true +length: 89 +--- no_error_log +[error] + + + +=== TEST 6: packet_parser finished +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local utils = require "resty.t1k.utils" + local finished, tag, length = utils.packet_parser("\xa0\x08\x00\x00\x00") + ngx.say("finished: ", finished) + ngx.say("tag == TAG_VERSION: ", tag == 32) + ngx.say("length: ", 8) + } + } +--- request +GET /t +--- response_body +finished: true +tag == TAG_VERSION: true +length: 8 +--- no_error_log +[error] + + + +=== TEST 7: starts_with +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local utils = require "resty.t1k.utils" + ngx.say("http://www.baidu.com starts with \"http\": ", utils.starts_with("http://www.baidu.com", "http")) + ngx.say("http://www.baidu.com starts with \"https\": ", utils.starts_with("http://www.baidu.com", "https")) + } + } +--- request +GET /t +--- response_body +http://www.baidu.com starts with "http": true +http://www.baidu.com starts with "https": false +--- no_error_log +[error] + + + +=== TEST 8: to_var_idx +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local utils = require "resty.t1k.utils" + local fmt = string.format + ngx.say(fmt("http_x_real_ip is %s and %s", utils.to_var_idx("http_x_real_ip"))) + ngx.say(fmt("X_REAL_IP is %s and %s", utils.to_var_idx("X_REAL_IP"))) + ngx.say(fmt("X-Forwarded-For: 1 is %s and %d", utils.to_var_idx("X-Forwarded-For: 1"))) + ngx.say(fmt("X-Forwarded-For: -1 is %s and %d", utils.to_var_idx("X-Forwarded-For: -1"))) + ngx.say(fmt("X-FORWARDED-FOR:100 is %s and %d", utils.to_var_idx("X-FORWARDED-FOR:-100"))) + } + } +--- request +GET /t +--- response_body +http_x_real_ip is http_x_real_ip and nil +X_REAL_IP is http_x_real_ip and nil +X-Forwarded-For: 1 is http_x_forwarded_for and 1 +X-Forwarded-For: -1 is http_x_forwarded_for and -1 +X-FORWARDED-FOR:100 is http_x_forwarded_for and -100 +--- no_error_log +[error] + + + +=== TEST 9: get_indexed_element +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local utils = require "resty.t1k.utils" + ngx.say("X-Forwarded-For: 1 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, 1)) + ngx.say("X-Forwarded-For: 2 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, 2)) + ngx.say("X-Forwarded-For: 3 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, 3)) + ngx.say("X-Forwarded-For: 4 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, 4)) + ngx.say("X-Forwarded-For: 5 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, 5)) + ngx.say("X-Forwarded-For: -1 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, -1)) + ngx.say("X-Forwarded-For: -2 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, -2)) + ngx.say("X-Forwarded-For: -3 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, -3)) + ngx.say("X-Forwarded-For: -4 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, -4)) + ngx.say("X-Forwarded-For: -5 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, -5)) + ngx.say("X-Forwarded-For: 0 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, 0)) + ngx.say("X-Non-Existent-Header is ", utils.get_indexed_element(ngx.var.http_non_existent_header)) + ngx.say("X-Real-IP is ", utils.get_indexed_element(ngx.var.http_x_real_ip)) + ngx.say("X-Real-IP: 1 is ", utils.get_indexed_element(ngx.var.http_x_real_ip, 1)) + ngx.say("X-Real-IP: 2 is ", utils.get_indexed_element(ngx.var.http_x_real_ip, 2)) + ngx.say("X-Real-IP: -1 is ", utils.get_indexed_element(ngx.var.http_x_real_ip, -1)) + ngx.say("X-Real-IP: -2 is ", utils.get_indexed_element(ngx.var.http_x_real_ip, -2)) + } + } +--- request +GET /t +--- more_headers +X-Forwarded-For: 1.1.1.1, 2.2.2.2 +X-Forwarded-For: 3.3.3.3, 4.4.4.4 +X-Real-IP: 10.10.10.10 +--- response_body +X-Forwarded-For: 1 is 1.1.1.1 +X-Forwarded-For: 2 is 2.2.2.2 +X-Forwarded-For: 3 is 3.3.3.3 +X-Forwarded-For: 4 is 4.4.4.4 +X-Forwarded-For: 5 is nil +X-Forwarded-For: -1 is 4.4.4.4 +X-Forwarded-For: -2 is 3.3.3.3 +X-Forwarded-For: -3 is 2.2.2.2 +X-Forwarded-For: -4 is 1.1.1.1 +X-Forwarded-For: -5 is nil +X-Forwarded-For: 0 is 1.1.1.1, 2.2.2.2, 3.3.3.3, 4.4.4.4 +X-Non-Existent-Header is nil +X-Real-IP is 10.10.10.10 +X-Real-IP: 1 is 10.10.10.10 +X-Real-IP: 2 is nil +X-Real-IP: -1 is 10.10.10.10 +X-Real-IP: -2 is nil +--- no_error_log +[error] + + + +=== TEST 10: get_event_id +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local utils = require "resty.t1k.utils" + ngx.say(utils.get_event_id("")) + ngx.say(utils.get_event_id("")) + ngx.say(utils.get_event_id("")) + } + } +--- request +GET /t +--- response_body +0988987de04844c7a3ce6d27865c9513 +8bae6adf33864c7f8bf715a9b7a65b2c +nil +--- no_error_log +[error] diff --git a/sdk/lua-resty-t1k/t/uuid.t b/sdk/lua-resty-t1k/t/uuid.t new file mode 100644 index 0000000..f1060b5 --- /dev/null +++ b/sdk/lua-resty-t1k/t/uuid.t @@ -0,0 +1,29 @@ +use Test::Nginx::Socket; + +our $HttpConfig = <<'_EOC_'; + lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; +_EOC_ + +repeat_each(3); + +plan tests => repeat_each() * (blocks() * 3); + +run_tests(); + +__DATA__ + +=== TEST 1: generate_v4 +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + local uuid = require "resty.t1k.uuid" + ngx.say(uuid.generate_v4()) + } + } +--- request +GET /t +--- response_body_like +^[0-9a-f]{8}[0-9a-f]{4}4[0-9a-f]{3}[89ab][0-9a-f]{3}[0-9a-f]{12}$ +--- no_error_log +[error] diff --git a/version.json b/version.json new file mode 100644 index 0000000..387ca1f --- /dev/null +++ b/version.json @@ -0,0 +1,5 @@ +{ + "latest_version": "v9.2.7", + "rec_version": "v9.2.7", + "lts_version": "v9.1.0-lts" +} diff --git a/yanshi/.gitignore b/yanshi/.gitignore new file mode 100644 index 0000000..a073c83 --- /dev/null +++ b/yanshi/.gitignore @@ -0,0 +1,8 @@ +*.o +*.dot +*.output +/build +/src/parser.cc +/src/parser.hh +/src/lexer.cc +/src/lexer.hh diff --git a/yanshi/Makefile b/yanshi/Makefile new file mode 100644 index 0000000..2d79e58 --- /dev/null +++ b/yanshi/Makefile @@ -0,0 +1,58 @@ +CPPFLAGS := -g3 -std=c++1y -Isrc -I. -DHAVE_READLINE + +ifeq ($(build),release) + BUILD := release + CPPFLAGS += -Os +else + BUILD := build + CPPFLAGS += -fsanitize=undefined,address -DDEBUG + LDLIBS := -lasan -lubsan +endif + +LDLIBS += -licuuc -lreadline +SRC := $(filter-out src/lexer.cc src/parser.cc, $(wildcard src/*.cc)) src/lexer.cc src/parser.cc +OBJ := $(addprefix $(BUILD)/,$(subst src/,,$(SRC:.cc=.o))) +UNITTEST_SRC := $(wildcard unittest/*.cc) +UNITTEST_EXE := $(subst unittest/,,$(UNITTEST_SRC:.cc=)) + +all: $(BUILD)/yanshi # unittest + +unittest: $(addprefix $(BUILD)/unittest/,$(UNITTEST_EXE)) + $(foreach x,$(addprefix $(BUILD)/unittest/,$(UNITTEST_EXE)),$x && ) : + +sinclude $(OBJ:.o=.d) + +# FIXME +$(BUILD)/repl.o: src/lexer.hh + +$(BUILD) $(BUILD)/unittest: + mkdir -p $@ + +$(BUILD)/yanshi: $(OBJ) + $(LINK.cc) $^ $(LDLIBS) -o $@ + +$(BUILD)/%.o: src/%.cc | $(BUILD) + $(CXX) $(CPPFLAGS) -MM -MP -MT $@ -MF $(@:.o=.d) $< + $(COMPILE.cc) $< -o $@ + +$(BUILD)/unittest/%: unittest/%.cc $(wildcard unittest/*.hh) $(filter-out $(BUILD)/main.o,$(OBJ)) | $(BUILD)/unittest + $(CXX) $(CPPFLAGS) -MM -MP -MT $@ -MF $(@:.o=.d) $< + $(LINK.cc) $(filter-out %.hh,$^) $(LDLIBS) -o $@ + +src/lexer.cc src/lexer.hh: src/lexer.l + flex --header-file=src/lexer.hh -o src/lexer.cc $< + +src/parser.cc src/parser.hh: src/parser.y src/common.hh src/location.hh src/option.hh src/syntax.hh + bison --defines=src/parser.hh -o src/parser.cc $< + +$(BUILD)/loader.o: src/parser.hh +$(BUILD)/parser.o: src/lexer.hh +$(BUILD)/lexer.o: src/parser.hh + +clean: + $(RM) -r build release + +distclean: clean + $(RM) src/{lexer,parser}.{cc,hh} + +.PHONY: all clean distclean diff --git a/yanshi/README.md b/yanshi/README.md new file mode 100644 index 0000000..e208dc5 --- /dev/null +++ b/yanshi/README.md @@ -0,0 +1,217 @@ +# 偃师 (yanshi) + +yanshi is a finite state automaton generator like Ragel. Use inline operators to embed C++ code in the recognition of a language. yanshi is enhanced with features to approximate context-free grammar: + +- Approximation of substring grammar +- Approximation of recursive automaton to match expressions with recursion. + +The motivation to create yanshi is that Ragel does not provide a mechanism to serialize its representation of finite state automata, making it difficult to post-process generated automata and obtain the substring grammar recognizer. + +Later on, I found a simplified SQL grammar might contain more than 10000 states. It was not only slow to generate the automaton, making it hard to do trial-and-error experiments, but wasted memory to store the automaton. I introduced `CollapseExpr` to allow circular references. + +`CallExpr` takes one step further, maintains a return address stack to imitate function calls. It can be seen as an augmented `CollapseExpr`, removing a lot of false positive cases. + +## Name + +From : + +
+In ancient China, a curious account of automata is found in the Lie Zi text (列子), written in the 3rd century BC. Within it there is a description of a much earlier encounter between King Mu of Zhou (周穆王, 1023-957 BC) and a mechanical engineer known as Yan Shi (偃师), an 'artificer'. +
+ +## Build + +* Debug: `make` +* Release: `make build=release` + +## Getting Started + +* Create file `a.ys`: + ``` + export foo = 'hello' + ``` + + Run `yanshi -S a.ys -o /tmp/a.cc` to generate a C++ file from the yanshi source file `a.ys`. + + + `yanshi_foo_start`: the start state is 0. States are represented by natural numbers. + + `yanshi_foo_is_final`: leave aside `ret_stack` and look at the last line, it checks whether `u` is one of the final states. + + `yanshi_foo_transit`: leave aside `ret_stack`, `u` is the current state and `c` is the next input codepoint or label. + + With the `-S` option, yanshi will generate a standalone C++ file. + ``` + % make -C /tmp a + make: Entering directory '/tmp' + g++ a.cc -o a + make: Leaving directory '/tmp' + % /tmp/a hello + 0 h 1 e 2 l 3 l 4 o 5 + len: 5 + pref: 5 + state: 5 + final: true + % /tmp/a + hello0 h 1 e 2 l 3 l 4 o 5 + len: 5 + pref: 5 + state: 5 + final: true + ``` + + States are yellow and interleaved with transition labels. Final states are bold yellow. + + `len`: the length of input codepoints or labels + + `pref`: the length of the longest prefix that does not enter the dead state + + `state`: the state entered after consuming the input + + `final`: whether the state is one of final states + +* Interactive mode + + The `-i` option enables interactive mode. + ``` + % yanshi -i a.ys + Testing foo + foo :: DefineStmt + .integer mode + Commands available from the prompt: + .automaton dump automaton + .assoc dump associated AST Expr for each state + .help display this help + .integer input is a list of non-negative integers, macros(#define) or '' quoted strings + .macro display defined macros + .string input is a string + .stmt change target DefineStmt to + .quit exit interactive mode + λ 104 101 108 108 111 + 0 104 1 101 2 108 3 108 4 111 5 + export foo = 'hello': + λ .string + .string mode + λ hello + 0 h 1 e 2 l 3 l 4 o 5 + export foo = 'hello': + λ + ``` + +* Regex-like syntax + ``` + export hello = [gh] 'e' l{2} 'o' + l = 'l' + ``` + + `[gh]` is a bracket expression and `l{2}` denotes to matches `l` at least twice. This grammar matches `hello`, `gello`, `helllo`, ... + + + Union: `c = a | b` + + Intersection: `c = a && b` + + Difference: `c = a - b` + + Concatenation: `c = a b` + + Complement: `c = ~ a` + +* Actions (embedded C++ code) + ``` + c++ { + #include + } + export hello = '喵' @ { puts("meow"); } {2} + ``` + I have not thought clearly on the implementation. The executing point may be counter-intuitive. + +* Modules + ``` + # a.ys + import 'b.ys' as B # B::bar + import 'b.ys' # qux + + export foo = B::bar | qux + bar = '4' + + # b.ys + bar = '3' + qux = '5' + ``` + +* Substring grammar + Specify the `--substring-grammar` option to generate code for substring grammar. That is, the generated code matches every substring of the grammar. The implementation creates a new start state and a new final state, connects the start state to the old start state, and old final states to the new final state. + +* `EmbedExpr`, reference a nonterminal without modifiers + ``` + foo = bar + bar = [0-9] + ``` + The complete automaton of bar will be duplicated in each reference site. If the referenced automaton is large, `EmbedExpr` will significantly increase the number of states. `EmbedExpr` defines dependencies among states and no cyclic dependency is allowed. + +* `CollapseExpr`, reference a nonterminal with the `!` modifier + ``` + export foo = 'pre' !bar 'post' + bar = [\u0300-\u034E] + quz = 'meow' !bar 'meow' + ``` + The final state of `'pre'` and the start state of `'post'` will be connected by a special directed arc. When exporting, an epsilon transition will be added from the tail of the arc to the start state of `bar`, others will be added from the final states of `bar` to the head of the arc. `CollapseExpr` behaves like function calls, however, the return address is not preserved (hence the name `CollapseExpr`) and the state may go to other call sites. In this example, the state may go to either `foo` or `quz` after traveling through `bar`, causing false positives. + +* `CallExpr`, reference a nonterminal with the `&` modifier + ``` + export foo = 'pre' &bar 'post' + bar = '4' + ``` + This is a refinement of `CollapseExpr`. Suppose state `&B` is contained in `A`'s definition (`A` calls `B`). `&B` will be represented as an pseudo arc (`u -> v`), where `u` is the state before `&B` and `v` is the state after `&B`. If arcs of `u` do not collide with arcs of `B`, the transition function will push `v` to the return stack if current state set contains `u` and there is no other transition. Note `B` is disconnected from `A`, which is different from the `CollapseExpr` case. The machine will transit on automaton `B` greedily. If there is no transition, it will pop a return address(`v` in this case) and jumps to it. + +## Contrib + +### Vim + +Syntax highlighting, and a syntax checking plugin for Synaptics + +```zsh +ln -sr contrib/vim/compiler/yanshi.vim ~/.vim/compiler/ +ln -sr contrib/vim/ftdetect/yanshi.vim ~/.vim/ftdetect/ +ln -sr contrib/vim/ftplugin/yanshi.vim ~/.vim/ftplugin/ +ln -sr contrib/vim/syntax/yanshi.vim ~/.vim/syntax/ +ln -sr contrib/vim/syntax_checker/yanshi ~/.vim/syntax_checker/ +``` + +### Zsh + +Command line completion + +``` +# ~/.zshrc +fpath=(~/.zsh $fpath) + +# ln -sr contrib/zsh/_yanshi ~/.zsh/ +``` + +## Internals + +``` +src + common.{cc,hh} + main.{cc,hh} + syntax.{cc,hh} + loader.{cc,hh} + fsa.{cc,hh} + fsa_anno.{cc,hh} + compiler.{cc,hh} + parser.y + lexer.l + location.cc +``` + +* Lex `lexer.l` +* Parse and generate a syntax tree `parser.y` +* `loader.cc` + + Get a list of definitions + + Recursively load for each `import` + + Resolve references and associate uses to definitions + + Build a dependency graph from `EmbedExpr` + + Compile automaton for each nonterminal in topological order. `CollapseExpr` and `CallExpr` are represented by special directed arcs. + + Generate code for `export` nonterminals, resolving `CollapseExpr` and `CallExpr` + +### Finite state automaton + +Each node of the syntax tree corresponds to an automaton. The parent builds an automaton from its children according to the semantics. The automaton of the parent may contain states from the automaton of one of the children, or it is a state introduced by the parent. + +`assoc[i]` records the associative nodes in the automaton tree (which part of the syntax tree has associations with this state) and positions (start state, final state or inner state) for state `i`. It serves three purposes: + +* Check which action should be triggered +* Look for inner states (neither start nor final) in the implementation of substring grammar +* Check whether it is associated to a `CallExpr` or `CollapseExpr` + +### `CollapseExpr` diff --git a/yanshi/contrib/vim/compiler/yanshi.vim b/yanshi/contrib/vim/compiler/yanshi.vim new file mode 100644 index 0000000..fcec7e8 --- /dev/null +++ b/yanshi/contrib/vim/compiler/yanshi.vim @@ -0,0 +1,15 @@ +"if exists('current_compiler') +" finish +"endif +let current_compiler = 'yanshi' + +if exists(':CompilerSet') != 2 + command -nargs=* CompilerSet setlocal +endif +CompilerSet errorformat= + \%E%f\ %l:%c-%*\\d\ error\ %m, + \%E%f\ %l-%*\\d:%c-%*\\d\ error\ %m, + \%W%f\ %l:%c-%*\\d\ warning\ %m, + \%W%f\ %l-%*\\d:%c-%*\\d\ warning\ %m, + \%C%.%# +CompilerSet makeprg=yanshi\ -d0\ -c\ $*\ % diff --git a/yanshi/contrib/vim/ftdetect/yanshi.vim b/yanshi/contrib/vim/ftdetect/yanshi.vim new file mode 100644 index 0000000..70224dc --- /dev/null +++ b/yanshi/contrib/vim/ftdetect/yanshi.vim @@ -0,0 +1,2 @@ +au BufRead,BufNewFile *.yanshi setf yanshi +au BufRead,BufNewFile *.ys setf yanshi diff --git a/yanshi/contrib/vim/ftplugin/yanshi.vim b/yanshi/contrib/vim/ftplugin/yanshi.vim new file mode 100644 index 0000000..0d66ffa --- /dev/null +++ b/yanshi/contrib/vim/ftplugin/yanshi.vim @@ -0,0 +1,6 @@ +if exists('b:did_ftplugin') + finish +endif +let b:did_ftplugin = 1 + +compiler yanshi diff --git a/yanshi/contrib/vim/syntax/yanshi.vim b/yanshi/contrib/vim/syntax/yanshi.vim new file mode 100644 index 0000000..22266f9 --- /dev/null +++ b/yanshi/contrib/vim/syntax/yanshi.vim @@ -0,0 +1,70 @@ +if exists('b:current_syntax') + finish +endif + +syn cluster yanshiCommentGroup contains=yanshiTodo +syn include @yanshiCcode syntax/cpp.vim +syn keyword yanshiAction action +syn keyword yanshiMacro semicolon nosemicolon +syn keyword yanshiStorageClass export intact +syn keyword yanshiTodo contained TODO FIXME XXX +syn match yanshiCpp 'c++' +syn match yanshiActionOperator '[>$@%]' +syn match yanshiCall '\^\w\+\(::\w\+\)\?' +syn match yanshiCollapse '!\w\+\(::\w\+\)\?' +syn match yanshiHighOp '[+\*?]' +syn match yanshiIdent '\w\+\(::\w\+\)\?' +syn match yanshiCpp display "^c++\s*" skipwhite nextgroup=yanshiBrace +syn match yanshiImport display "^import\s*" contains=yanshiImported +syn match yanshiLowOp '[-&|]' +syn match yanshiSpecial display contained "\\\(x\x\x\|.\|$\)" +syn region yanshiBrace matchgroup=Delimiter start='{' end='}' fold contains=@yanshiCcode +syn region yanshiBracket start='\[' skip=+\\\\\|\\]+ end=']' +syn region yanshiComment start='/\*' end='\*/' keepend contains=@yanshiCommentGroup,@Spell +syn region yanshiImported display contained start="+" skip=+\\\\\|\\"+ end=+"+ +syn region yanshiLineComment start='#\|//' skip='\\$' end='$' keepend contains=@yanshiCommentGroup,@Spell +syn region yanshiPreprocess start="#define" skip="\\$" end="$" keepend +syn region yanshiQQString start=+"+ skip=+\\.+ end=+"+ contains=yanshiSpecial +syn region yanshiQString start=+'+ skip=+\\.+ end=+'+ + +syn region yanshiDefineStmt start='^\w\+\s*[=:]' end='$' skipnl contains=@yanshiExpr,yanshiComment,yanshiLineComment,yanshiParen0 + +syn cluster yanshiExpr contains=yanshiActionOperator,yanshiBrace,yanshiBracket,yanshiCall,yanshiCollapse,yanshiIdent,yanshiHighOp,yanshiLowOp,yanshiQString,yanshiQQString, +sy region yanshiParen0 matchgroup=yanshiParen0 start='(' end=')' contains=@yanshiExpr,yanshiParen1 +sy region yanshiParen1 matchgroup=yanshiParen1 start='(' end=')' contains=@yanshiExpr,yanshiParen2 contained +sy region yanshiParen2 matchgroup=yanshiParen2 start='(' end=')' contains=@yanshiExpr,yanshiParen3 contained +sy region yanshiParen3 matchgroup=yanshiParen3 start='(' end=')' contains=@yanshiExpr,yanshiParen4 contained +sy region yanshiParen4 matchgroup=yanshiParen4 start='(' end=')' contains=@yanshiExpr,yanshiParen5 contained +sy region yanshiParen5 matchgroup=yanshiParen5 start='(' end=')' contains=@yanshiExpr,yanshiParen0 contained +hi yanshiParen0 ctermfg=brown guifg=#3bb9ff +hi yanshiParen1 ctermfg=DarkBlue guifg=#f88017 +hi yanshiParen2 ctermfg=darkgray guifg=#5efb6e +hi yanshiParen3 ctermfg=darkgreen guifg=#f62817 +hi yanshiParen4 ctermfg=darkcyan guifg=#fdd017 +hi yanshiParen5 ctermfg=darkmagenta guifg=#faafba + +hi link yanshiIdent Identifier +"TODO color mismatch of {} +"hi link yanshiBrace Statement +"hi link yanshiDefineStmt Statement +hi def link yanshiCall Constant +hi def link yanshiCollapse Constant +hi def link yanshiAction Structure +hi def link yanshiActionOperator Type +hi def link yanshiBracket Function +hi def link yanshiCpp Structure +hi def link yanshiComment Comment +hi def link yanshiHighOp Operator +hi def link yanshiImport Include +hi def link yanshiImported String +hi def link yanshiLineComment Comment +hi def link yanshiLowOp Conditional +hi def link yanshiMacro Macro +hi def link yanshiPreprocess Macro +hi def link yanshiQQString String +hi def link yanshiQString String +hi def link yanshiSpecial SpecialChar +hi def link yanshiStorageClass StorageClass +hi def link yanshiTodo Todo + +let b:current_syntax = 'yanshi' diff --git a/yanshi/contrib/vim/syntax_checkers/yanshi/yanshi.vim b/yanshi/contrib/vim/syntax_checkers/yanshi/yanshi.vim new file mode 100644 index 0000000..da85c41 --- /dev/null +++ b/yanshi/contrib/vim/syntax_checkers/yanshi/yanshi.vim @@ -0,0 +1,29 @@ +if exists('g:loaded_syntastic_yanshi_yanshi_checker') + finish +endif +let g:loaded_syntastic_yanshi_yanshi_checker = 1 + +let s:save_cpo = &cpo +set cpo&vim + +fu! SyntaxCheckers_yanshi_yanshi_GetLocList() dict + let makeprg = self.makeprgBuild({ 'args': '-d0 -c' }) + + let errorformat = + \ '%C %.%#,'. + \ '%E%f %l:%c-%*\d error %m,'. + \ '%E%f %l-%*\d:%c-%*\d error %m,'. + \ '%W%f %l:%c-%*\d warning %m,'. + \ '%W%f %l-%*\d:%c-%*\d warning %m' + + return SyntasticMake({ + \ 'makeprg': makeprg, + \ 'errorformat': errorformat }) +endf + +call g:SyntasticRegistry.CreateAndRegisterChecker({ + \ 'filetype': 'yanshi', + \ 'name': 'yanshi'}) + +let &cpo = s:save_cpo +unlet s:save_cpo diff --git a/yanshi/contrib/zsh/_yanshi b/yanshi/contrib/zsh/_yanshi new file mode 100644 index 0000000..f65e4bd --- /dev/null +++ b/yanshi/contrib/zsh/_yanshi @@ -0,0 +1,24 @@ +#compdef yanshi + +_arguments \ + '(-b --bytes)'{-b,--bytes}'[make labels range over \[0,256), Unicode literals will be treated as UTF-8 bytes]' \ + '(-c --check)'{-c,--check}'[check syntax & use/def]' \ + '-C[generate C source code (default: C++)]' \ + '(-d --debug)'{-d,--debug}'+[debug level]:level:(0 1 2 3 4 5)' \ + '--dump-action[dump associated actions for each edge]' \ + '--dump-assoc[dump associated AST Expr for each state]' \ + '--dump-automaton[dump automata]' \ + '--dump-embed[dump statistics of EmbedExpr]' \ + '--dump-module[dump module use/def/...]' \ + '--dump-tree[dump AST]' \ + '(-G --graph)'{-G,--graph}'[output a Graphviz dot file]' \ + '(-I --import)'{-I,--import}'=[add to search path for "import"]' \ + '(-i --interactive)'{-i,--interactive}'[interactive mode]' \ + '(-k --keep-inaccessible)'{-k,--keep-inaccessible}'[do not perform accessible/co-accessible]' \ + '(-l --debug-output)'{-l,--debug-output}'=[filename for debug output]:file:_files' \ + '--max-return-stack=[max length of return stack in C generator]:len:' \ + '(-o --output)'{-o,--output}'=[.cc output filename]:file:_files' \ + '(-O --output-header)'{-O,--output-header}'=[.hh output filename]:file:_files' \ + '(-s --substring-grammar)'{-s,--substring-grammar}'[construct regular approximation of the substring grammar. Inner states of nonterminals labeled 'intact' are not connected to start/final]' \ + '(-h --help)'{-h,--help}'[display this help]' \ + '1:file:_files -g "*.{ys,yanshi}"'\ diff --git a/yanshi/src/common.cc b/yanshi/src/common.cc new file mode 100644 index 0000000..2ea555b --- /dev/null +++ b/yanshi/src/common.cc @@ -0,0 +1,187 @@ +#include "common.hh" +#include "option.hh" + +#include +#include +#include +#include +#include +#include +#include +#include + +///// Error + +static const char *ENAME[] = { + /* 0 */ "", + /* 1 */ "EPERM", "ENOENT", "ESRCH", "EINTR", "EIO", "ENXIO", + /* 7 */ "E2BIG", "ENOEXEC", "EBADF", "ECHILD", + /* 11 */ "EAGAIN/EWOULDBLOCK", "ENOMEM", "EACCES", "EFAULT", + /* 15 */ "ENOTBLK", "EBUSY", "EEXIST", "EXDEV", "ENODEV", + /* 20 */ "ENOTDIR", "EISDIR", "EINVAL", "ENFILE", "EMFILE", + /* 25 */ "ENOTTY", "ETXTBSY", "EFBIG", "ENOSPC", "ESPIPE", + /* 30 */ "EROFS", "EMLINK", "EPIPE", "EDOM", "ERANGE", + /* 35 */ "EDEADLK/EDEADLOCK", "ENAMETOOLONG", "ENOLCK", "ENOSYS", + /* 39 */ "ENOTEMPTY", "ELOOP", "", "ENOMSG", "EIDRM", "ECHRNG", + /* 45 */ "EL2NSYNC", "EL3HLT", "EL3RST", "ELNRNG", "EUNATCH", + /* 50 */ "ENOCSI", "EL2HLT", "EBADE", "EBADR", "EXFULL", "ENOANO", + /* 56 */ "EBADRQC", "EBADSLT", "", "EBFONT", "ENOSTR", "ENODATA", + /* 62 */ "ETIME", "ENOSR", "ENONET", "ENOPKG", "EREMOTE", + /* 67 */ "ENOLINK", "EADV", "ESRMNT", "ECOMM", "EPROTO", + /* 72 */ "EMULTIHOP", "EDOTDOT", "EBADMSG", "EOVERFLOW", + /* 76 */ "ENOTUNIQ", "EBADFD", "EREMCHG", "ELIBACC", "ELIBBAD", + /* 81 */ "ELIBSCN", "ELIBMAX", "ELIBEXEC", "EILSEQ", "ERESTART", + /* 86 */ "ESTRPIPE", "EUSERS", "ENOTSOCK", "EDESTADDRREQ", + /* 90 */ "EMSGSIZE", "EPROTOTYPE", "ENOPROTOOPT", + /* 93 */ "EPROTONOSUPPORT", "ESOCKTNOSUPPORT", + /* 95 */ "EOPNOTSUPP/ENOTSUP", "EPFNOSUPPORT", "EAFNOSUPPORT", + /* 98 */ "EADDRINUSE", "EADDRNOTAVAIL", "ENETDOWN", "ENETUNREACH", + /* 102 */ "ENETRESET", "ECONNABORTED", "ECONNRESET", "ENOBUFS", + /* 106 */ "EISCONN", "ENOTCONN", "ESHUTDOWN", "ETOOMANYREFS", + /* 110 */ "ETIMEDOUT", "ECONNREFUSED", "EHOSTDOWN", "EHOSTUNREACH", + /* 114 */ "EALREADY", "EINPROGRESS", "ESTALE", "EUCLEAN", + /* 118 */ "ENOTNAM", "ENAVAIL", "EISNAM", "EREMOTEIO", "EDQUOT", + /* 123 */ "ENOMEDIUM", "EMEDIUMTYPE", "ECANCELED", "ENOKEY", + /* 127 */ "EKEYEXPIRED", "EKEYREVOKED", "EKEYREJECTED", + /* 130 */ "EOWNERDEAD", "ENOTRECOVERABLE", "ERFKILL", "EHWPOISON" +}; + +#define MAX_ENAME 133 + +long action_label_base, action_label, call_label_base, call_label, collapse_label_base, collapse_label; + +void output_error(bool use_err, const char *format, va_list ap) +{ + char text[BUF_SIZE], msg[BUF_SIZE], buf[BUF_SIZE]; + vsnprintf(msg, BUF_SIZE, format, ap); + if (use_err) + snprintf(text, BUF_SIZE, "[%s %s] ", 0 < errno && errno < MAX_ENAME ? ENAME[errno] : "?UNKNOWN?", strerror(errno)); + else + strcpy(text, ""); + snprintf(buf, BUF_SIZE, RED "%s%s\n", text, msg); + fputs(buf, stderr); + fputs(SGR0, stderr); + fflush(stderr); +} + +void err_msg(const char *format, ...) +{ + va_list ap; + va_start(ap, format); + int saved = errno; + output_error(errno > 0, format, ap); + errno = saved; + va_end(ap); +} +#define err_msg_g(...) ({err_msg(__VA_ARGS__); goto quit;}) + +void err_exit(int exitno, const char *format, ...) +{ + va_list ap; + va_start(ap, format); + int saved = errno; + output_error(errno > 0, format, ap); + errno = saved; + va_end(ap); + + void *bt[99]; + char buf[1024]; + int nptrs = backtrace(bt, LEN(buf)); + int i = sprintf(buf, "addr2line -Cfipe %s", program_invocation_name), j = 0; + while (j < nptrs && i+30 < sizeof buf) + i += sprintf(buf+i, " %p", bt[j++]); + strcat(buf, ">&2"); + fputs("\n", stderr); + system(buf); + //backtrace_symbols_fd(buf, nptrs, STDERR_FILENO); + exit(exitno); +} + +long get_long(const char *arg) +{ + char *end; + errno = 0; + long ret = strtol(arg, &end, 0); + if (errno) + err_exit(EX_USAGE, "get_long: %s", arg); + if (*end) + err_exit(EX_USAGE, "get_long: nonnumeric character"); + return ret; +} + +//// log +// + +void log_generic(const char *prefix, const char *format, va_list ap) +{ + char buf[BUF_SIZE]; + timeval tv; + tm tm; + gettimeofday(&tv, NULL); + fputs(prefix, stdout); + if (localtime_r(&tv.tv_sec, &tm)) { + strftime(buf, sizeof buf, "%T.%%06u ", &tm); + printf(buf, tv.tv_usec); + } + vprintf(format, ap); + fputs(SGR0, stdout); + fflush(stdout); +} + +void log_event(const char *format, ...) +{ + va_list ap; + va_start(ap, format); + log_generic(CYAN, format, ap); + va_end(ap); +} + +void log_action(const char *format, ...) +{ + va_list ap; + va_start(ap, format); + log_generic(GREEN, format, ap); + va_end(ap); +} + +void log_status(const char *format, ...) +{ + va_list ap; + va_start(ap, format); + log_generic(YELLOW, format, ap); + va_end(ap); +} + +void bold(long fd) { if (isatty(fd)) fputs("\x1b[1m", fd == STDOUT_FILENO ? stdout : stderr); } +void blue(long fd) { if (isatty(fd)) fputs(BLUE, fd == STDOUT_FILENO ? stdout : stderr); } +void cyan(long fd) { if (isatty(fd)) fputs(CYAN, fd == STDOUT_FILENO ? stdout : stderr); } +void green(long fd) { if (isatty(fd)) fputs(GREEN, fd == STDOUT_FILENO ? stdout : stderr); } +void magenta(long fd) { if (isatty(fd)) fputs(MAGENTA, fd == STDOUT_FILENO ? stdout : stderr); } +void red(long fd) { if (isatty(fd)) fputs(RED, fd == STDOUT_FILENO ? stdout : stderr); } +void sgr0(long fd) { if (isatty(fd)) fputs(SGR0, fd == STDOUT_FILENO ? stdout : stderr); } +void yellow(long fd) { if (isatty(fd)) fputs(YELLOW, fd == STDOUT_FILENO ? stdout : stderr); } +void normal_yellow(long fd) { if (isatty(fd)) fputs(NORMAL_YELLOW, fd == STDOUT_FILENO ? stdout : stderr); } + +void indent(FILE* f, int d) +{ + fprintf(f, "%*s", 2*d, ""); +} + +void DisjointIntervals::flip() { + long i = 0; + map to2; + for (auto &x: to) { + if (i < x.first) + to2.emplace(i, x.first); + i = x.second; + } + if (i < AB) + to2.emplace(i, AB); + to = move(to2); +} + +void DisjointIntervals::print() { + for (auto& x: to) + printf("(%ld,%ld) ", x.first, x.second); + puts(""); +} diff --git a/yanshi/src/common.hh b/yanshi/src/common.hh new file mode 100644 index 0000000..1d1051b --- /dev/null +++ b/yanshi/src/common.hh @@ -0,0 +1,130 @@ +#pragma once +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif +#include +#include +#include +#include +#include +#include +#include +using std::map; +using std::vector; + +typedef int8_t i8; +typedef int16_t i16; +typedef int32_t i32; +typedef int64_t i64; +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +typedef unsigned long ulong; + +#ifdef __APPLE__ +#include +extern char*** _NSGetArgv(void); +#define program_invocation_name (((char **)*_NSGetArgv())[0]) +#define program_invocation_short_name (((char **)*_NSGetArgv())[0]) +#endif + +#define LEN(x) (sizeof(x)/sizeof(*x)) +#define ALL(x) (x).begin(), (x).end() +#define REP(i, n) FOR(i, 0, n) +#define FOR(i, a, b) for (typename std::remove_cv::type>::type i = (a); i < (b); i++) +#define ROF(i, a, b) for (typename std::remove_cv::type>::type i = (b); --i >= (a); ) + +#define SGR0 "\x1b[m" +#define RED "\x1b[1;31m" +#define GREEN "\x1b[1;32m" +#define YELLOW "\x1b[1;33m" +#define BLUE "\x1b[1;34m" +#define MAGENTA "\x1b[1;35m" +#define CYAN "\x1b[1;36m" +#define NORMAL_YELLOW "\x1b[33m" +const long MAX_CODEPOINT = 0x10ffff; +extern long action_label_base, action_label, call_label_base, call_label, collapse_label_base, collapse_label; + +void bold(long fd = 1); +void blue(long fd = 1); +void cyan(long fd = 1); +void green(long fd = 1); +void magenta(long fd = 1); +void red(long fd = 1); +void sgr0(long fd = 1); +void yellow(long fd = 1); +void normal_yellow(long fd = 1); +void indent(FILE* f, int d); + +const size_t BUF_SIZE = 512; + +void output_error(bool use_err, const char *format, va_list ap); +void err_msg(const char *format, ...); +void err_exit(int exitno, const char *format, ...); + +long get_long(const char *arg); + +void log_generic(const char *prefix, const char *format, va_list ap); +void log_event(const char *format, ...); +void log_action(const char *format, ...); +void log_status(const char *format, ...); + +extern long debug_level; +extern FILE* debug_file; +#define DP(level, ...) do { \ + if (level <= debug_level) { \ + fprintf(debug_file, "%s:%d ", __FILE__, __LINE__); \ + fprintf(debug_file, __VA_ARGS__);\ + fprintf(debug_file, "\n"); \ + fflush(debug_file); \ + } \ + } while (0) + +template +void emplace_front(vector& a, Args&&... args) +{ + a.emplace(a.begin(), args...); +} + +template +void sorted_insert(vector& a, const T& x) +{ + a.emplace_back(); + auto it = a.end(); + while (a.begin() != --it && x < it[-1]) + *it = it[-1]; + *it = x; +} + +template +void sorted_emplace(vector& a, Args&&... args) +{ + T x{args...}; + a.emplace_back(); + auto it = a.end(); + while (a.begin() != --it && x < it[-1]) + *it = it[-1]; + *it = x; +} + +struct DisjointIntervals +{ + typedef std::pair value_type; + std::map to; + template + void emplace(Args&&... args) { + value_type x{args...}; + auto it = to.lower_bound(x.first); + if (it != to.begin() && x.first <= prev(it)->second) + x.first = (--it)->first; + auto it2 = to.upper_bound(x.second); + if (it2 != to.begin() && prev(it2)->first <= x.second && x.second < prev(it2)->second) + x.second = prev(it2)->second; + while (it != it2) + it = to.erase(it); + to.emplace(x); + } + void flip(); + void print(); // XXX +}; diff --git a/yanshi/src/compiler.cc b/yanshi/src/compiler.cc new file mode 100644 index 0000000..809961c --- /dev/null +++ b/yanshi/src/compiler.cc @@ -0,0 +1,853 @@ +#include "compiler.hh" +#include "fsa_anno.hh" +#include "loader.hh" +#include "option.hh" + +#include +#include +#include +#include +#include +#include +#include +using namespace std; + +unordered_map compiled; +static unordered_map>> stmt2call_addr; +static unordered_map> stmt2final; + +void print_assoc(const FsaAnno& anno) +{ + magenta(); printf("=== Associated Expr of each state\n"); sgr0(); + REP(i, anno.fsa.n()) { + printf("%ld:", i); + for (auto aa: anno.assoc[i]) { + auto a = aa.first; + printf(" %s%s%s%s(%ld-%ld", a->name().c_str(), + has_start(aa.second) ? "^" : "", + has_inner(aa.second) ? "." : "", + has_final(aa.second) ? "$" : "", + a->loc.start, a->loc.end); + if (a->entering.size()) + printf(",>%zd", a->entering.size()); + if (a->leaving.size()) + printf(",%%%zd", a->leaving.size()); + if (a->finishing.size()) + printf(",@%zd", a->finishing.size()); + if (a->transiting.size()) + printf(",$%zd", a->transiting.size()); + printf(")"); + } + puts(""); + } + puts(""); +} + +void print_automaton(const Fsa& fsa) +{ + magenta(); printf("=== Automaton\n"); sgr0(); + green(); printf("start: %ld\n", fsa.start); + red(); printf("finals:"); + for (long i: fsa.finals) + printf(" %ld", i); + puts(""); + sgr0(); puts("edges:"); + REP(i, fsa.n()) { + printf("%ld:", i); + for (auto it = fsa.adj[i].begin(); it != fsa.adj[i].end(); ) { + long from = it->first.first, to = it->first.second, v = it->second; + while (++it != fsa.adj[i].end() && to == it->first.first && it->second == v) + to = it->first.second; + if (from == to-1) + printf(" (%ld,%ld)", from, v); + else + printf(" (%ld-%ld,%ld)", from, to-1, v); + } + puts(""); + } + puts(""); +} + +Expr* find_lca(Expr* u, Expr* v) +{ + if (u->depth > v->depth) + swap(u, v); + if (u->depth < v->depth) + for (long k = 63-__builtin_clzl(v->depth-u->depth); k >= 0; k--) + if (u->depth <= v->depth-(1L<anc[k]; + if (u == v) + return u; + if (v->depth) + for (long k = 63-__builtin_clzl(v->depth); k >= 0; k--) + if (k < u->anc.size() && u->anc[k] != v->anc[k]) + u = u->anc[k], v = v->anc[k]; + return u->anc[0]; // NULL if two trees +} + +struct Compiler : Visitor { + stack st; + stack path; + long tick = 0; + + void pre_expr(Expr& expr) { + expr.pre = tick++; + expr.depth = path.size(); + if (path.size()) { + expr.anc.assign(1, path.top()); + for (long k = 1; 1L << k <= expr.depth; k++) + expr.anc.push_back(expr.anc[k-1]->anc[k-1]); + } else + expr.anc.assign(1, nullptr); + path.push(&expr); + DP(5, "%s(%ld-%ld)", expr.name().c_str(), expr.loc.start, expr.loc.end); + } + void post_expr(Expr& expr) { + path.pop(); + expr.post = tick; +#ifdef DEBUG + st.top().fsa.check(); +#endif + } + + void visit(Expr& expr) override { + pre_expr(expr); + expr.accept(*this); + post_expr(expr); + } + void visit(BracketExpr& expr) override { + st.push(FsaAnno::bracket(expr)); + } + void visit(CallExpr& expr) override { + st.push(FsaAnno::call(expr)); + } + void visit(CollapseExpr& expr) override { + st.push(FsaAnno::collapse(expr)); + } + void visit(ComplementExpr& expr) override { + visit(*expr.inner); + st.top().complement(&expr); + } + void visit(ConcatExpr& expr) override { + visit(*expr.rhs); + FsaAnno rhs = move(st.top()); + visit(*expr.lhs); + st.top().concat(rhs, &expr); + } + void visit(DifferenceExpr& expr) override { + visit(*expr.rhs); + FsaAnno rhs = move(st.top()); + visit(*expr.lhs); + st.top().difference(rhs, &expr); + } + void visit(DotExpr& expr) override { + st.push(FsaAnno::dot(&expr)); + } + void visit(EmbedExpr& expr) override { + st.push(FsaAnno::embed(expr)); + } + void visit(EpsilonExpr& expr) override { + st.push(FsaAnno::epsilon_fsa(&expr)); + } + void visit(IntersectExpr& expr) override { + visit(*expr.rhs); + FsaAnno rhs = move(st.top()); + visit(*expr.lhs); + st.top().intersect(rhs, &expr); + } + void visit(LiteralExpr& expr) override { + st.push(FsaAnno::literal(expr)); + } + void visit(PlusExpr& expr) override { + visit(*expr.inner); + st.top().plus(&expr); + } + void visit(QuestionExpr& expr) override { + visit(*expr.inner); + st.top().question(&expr); + } + void visit(RepeatExpr& expr) override { + visit(*expr.inner); + st.top().repeat(expr); + } + void visit(StarExpr& expr) override { + visit(*expr.inner); + st.top().star(&expr); + } + void visit(UnionExpr& expr) override { + visit(*expr.rhs); + FsaAnno rhs = move(st.top()); + visit(*expr.lhs); + st.top().union_(rhs, &expr); + } +}; + +void compile(DefineStmt* stmt) +{ + if (compiled.count(stmt)) + return; + FsaAnno& anno = compiled[stmt]; + Compiler comp; + comp.visit(*stmt->rhs); + anno = move(comp.st.top()); + anno.determinize(NULL, NULL); + anno.minimize(NULL); + DP(4, "size(%s::%s) = %ld", stmt->module->filename.c_str(), stmt->lhs.c_str(), anno.fsa.n()); +} + +void generate_transitions(DefineStmt* stmt) +{ + FsaAnno& anno = compiled[stmt]; + auto& call_addr = stmt2call_addr[stmt]; + auto& sub_final = stmt2final[stmt]; + auto find_within = [&](long u) { + vector> within; + Expr* last = NULL; + sort(ALL(anno.assoc[u]), [](const pair& x, const pair& y) { + if (x.first->pre != y.first->pre) + return x.first->pre < y.first->pre; + return x.second < y.second; + }); + for (auto aa: anno.assoc[u]) { + Expr* stop = last ? find_lca(last, aa.first) : NULL; + last = aa.first; + for (Expr* x = aa.first; x != stop; x = x->anc[0]) + within.emplace_back(x, aa.second); + } + sort(ALL(within)); + auto j = within.begin(); + for (auto i = within.begin(); i != within.end(); ) { + Expr* x = i->first; + long t = long(i->second); + while (++i != within.end() && x == i->first) + t |= long(i->second); + *j++ = {x, ExprTag(t)}; + } + within.erase(j, within.end()); + return within; + }; + decltype(anno.assoc) withins(anno.fsa.n()); + REP(i, anno.fsa.n()) + withins[i] = move(find_within(i)); + + auto get_code = [](Action* action) { + if (auto t = dynamic_cast(action)) + return t->code; + else if (auto t = dynamic_cast(action)) + return t->define_stmt->code; + else + assert(0); + return string(); + }; + +#define D(S) if (opt_dump_action) { \ + if (auto t = dynamic_cast(action.first)) { \ + if (from == to-1) \ + printf(S " %ld %ld %ld %s\n", u, from, v, t->code.c_str()); \ + else \ + printf(S " %ld %ld-%ld %ld %s\n", u, from, to-1, v, t->code.c_str()); \ + } else if (auto t = dynamic_cast(action.first)) { \ + if (from == to-1) \ + printf(S " %ld %ld %ld %s\n", u, from, v, t->define_stmt->code.c_str()); \ + else \ + printf(S " %ld %ld-%ld %ld %s\n", u, from, to-1, v, t->define_stmt->code.c_str()); \ + } \ + } + + if (output_header) { + if (opt_gen_c) { + if (opt_gen_extern_c) + fputs("extern \"C\" ", output_header); + fprintf(output_header, "long yanshi_%s_transit(long* ret_stack, long* ret_stack_len, long u, long c", stmt->lhs.c_str()); + } + else + fprintf(output_header, "long yanshi_%s_transit(vector& ret_stack, long u, long c", stmt->lhs.c_str()); + if (stmt->export_params.size()) + fprintf(output_header, ", %s", stmt->export_params.c_str()); + fprintf(output_header, ");\n"); + } + if (opt_gen_c) { + if (opt_gen_extern_c) + fputs("extern \"C\" ", output); + fprintf(output, "long yanshi_%s_transit(long* ret_stack, long* ret_stack_len, long u, long c", stmt->lhs.c_str()); + } + else + fprintf(output, "long yanshi_%s_transit(vector& ret_stack, long u, long c", stmt->lhs.c_str()); + if (stmt->export_params.size()) + fprintf(output, ", %s", stmt->export_params.c_str()); +fprintf(output, +")\n" +"{\n" +" long v = -1;\n" +"again:\n" +" switch (u) {\n"); + REP(u, anno.fsa.n()) { + if (call_addr[u].first >= 0) { // no other transitions + fprintf(output, +" case %ld:\n" +" u = %ld;\n" +, u, call_addr[u].first); + if (opt_gen_c) + fprintf(output, +" if (*ret_stack_len >= %ld) return -1;\n" +" ret_stack[(*ret_stack_len)++] = %ld;\n" +, opt_max_return_stack, call_addr[u].second); + else + fprintf(output, +" ret_stack.push_back(%ld);\n" +, call_addr[u].second); + fprintf(output, +" goto again;\n"); + continue; + } + if (anno.fsa.adj[u].empty() && ! sub_final[u]) + continue; + indent(output, 1); + fprintf(output, "case %ld:\n", u); + indent(output, 2); + fprintf(output, "switch (c) {\n"); + + unordered_map>, vector>>> v2case; + for (auto it = anno.fsa.adj[u].begin(); it != anno.fsa.adj[u].end(); ) { + long from = it->first.first, to = it->first.second, v = it->second; + while (++it != anno.fsa.adj[u].end() && to == it->first.first && it->second == v) + to = it->first.second; + v2case[v].first.emplace_back(from, to); + auto& body = v2case[v].second; + + auto ie = withins[u].end(), je = withins[v].end(); + + // leaving = Expr(u) - Expr(v) + for (auto i = withins[u].begin(), j = withins[v].begin(); i != ie; ++i) { + while (j != je && i->first > j->first) + ++j; + if (j == je || i->first != j->first) + for (auto action: i->first->leaving) { + D("%%"); + body.push_back(action); + } + } + + // entering = Expr(v) - Expr(u) + for (auto i = withins[u].begin(), j = withins[v].begin(); j != je; ++j) { + while (i != ie && i->first < j->first) + ++i; + if (i == ie || i->first != j->first) + for (auto action: j->first->entering) { + D(">"); + body.push_back(action); + } + } + + // transiting = intersect(Expr(u), Expr(v)) + for (auto i = withins[u].begin(), j = withins[v].begin(); j != je; ++j) { + while (i != ie && i->first < j->first) + ++i; + if (i != ie && i->first == j->first) + for (auto action: j->first->transiting) { + D("$"); + body.push_back(action); + } + } + + // finishing = intersect(Expr(u), Expr(v)) & Expr(v).has_final(v) + for (auto i = withins[u].begin(), j = withins[v].begin(); j != je; ++j) { + while (i != ie && i->first < j->first) + ++i; + if (i != ie && i->first == j->first && has_final(j->second)) + for (auto action: j->first->finishing) { + D("@"); + body.push_back(action); + } + } + } + + for (auto& x: v2case) { + for (auto& y: x.second.first) { + indent(output, 2); + if (y.first == y.second-1) + fprintf(output, "case %ld:\n", y.first); + else + fprintf(output, "case %ld ... %ld:\n", y.first, y.second-1); + } + indent(output, 3); + fprintf(output, "v = %ld;\n", x.first); + + // actions + sort(ALL(x.second.second), [](const pair& a0, const pair& a1) { + return a0.second != a1.second ? a0.second < a1.second : a0.first < a1.first; + }); + x.second.second.erase(unique(ALL(x.second.second)), x.second.second.end()); + for (auto a: x.second.second) + fprintf(output, "{%s}\n", get_code(a.first).c_str()); + indent(output, 3); + fprintf(output, "break;\n"); + } + // return from finals of DefineStmt called by CallExpr + if (sub_final[u]) { + indent(output, 2); + fprintf(output, "default:\n"); + indent(output, 3); + fprintf(output, opt_gen_c ? +"if (*ret_stack_len) { u = ret_stack[--*ret_stack_len]; goto again; }\n" +: +"if (ret_stack.size()) { u = ret_stack.back(); ret_stack.pop_back(); goto again; }\n"); + indent(output, 3); + fprintf(output, "break;\n"); + } + + indent(output, 2); + fprintf(output, "}\n"); + indent(output, 2); + fprintf(output, "break;\n"); + } + indent(output, 1); + fprintf(output, "}\n"); + indent(output, 1); + fprintf(output, "return v;\n"); + fprintf(output, "}\n\n"); +} + +bool compile_export(DefineStmt* stmt) +{ + DP(2, "Exporting %s", stmt->lhs.c_str()); + FsaAnno& anno = compiled[stmt]; + + DP(3, "Construct automaton with all DefineStmt associated to referenced CallExpr/CollapseExpr"); + vector> adj; + decltype(anno.assoc) assoc; + vector> cllps; + long allo = 0; + unordered_map stmt2offset; + unordered_map stmt2start; + unordered_map start2stmt; + vector starts; + vector sub_final; + function allocate = [&](DefineStmt* stmt) { + if (stmt2offset.count(stmt)) + return; + DP(4, "Allocate %ld to %s", allo, stmt->lhs.c_str()); + FsaAnno& anno = compiled[stmt]; + long base = stmt2offset[stmt] = allo; + allo += anno.fsa.n(); + sub_final.resize(allo); + if (used_as_call.count(stmt)) { + stmt2start[stmt] = base+anno.fsa.start; + start2stmt[base+anno.fsa.start] = stmt; + starts.push_back(base+anno.fsa.start); + for (long f: anno.fsa.finals) + sub_final[base+f] = true; + } + adj.insert(adj.end(), ALL(anno.fsa.adj)); + REP(i, anno.fsa.n()) + for (auto& e: adj[base+i]) + e.second += base; + assoc.insert(assoc.end(), ALL(anno.assoc)); + FOR(i, base, base+anno.fsa.n()) { + for (auto aa: assoc[i]) + if (has_start(aa.second)) { + if (auto* e = dynamic_cast(aa.first)) { + DefineStmt* v = e->define_stmt; + allocate(v); + } else if (auto* e = dynamic_cast(aa.first)) { + DefineStmt* v = e->define_stmt; + allocate(v); + // (i@{CollapseExpr,...}, special, _) -> ({CollapseExpr,...}, epsilon, CollapseExpr.define_stmt.start) + sorted_emplace(adj[i], epsilon, stmt2offset[v]+compiled[v].fsa.start); + } + } + long j = adj[i].size(); + while (j && collapse_label_base < adj[i][j-1].first.second) { + long v = adj[i][j-1].second; + if (adj[i][j-1].first.first < collapse_label_base) + adj[i][j-1].first.second = collapse_label_base; + else + j--; + CollapseExpr* e; + for (auto aa: assoc[v]) + if (has_final(aa.second) && (e = dynamic_cast(aa.first))) { + DefineStmt* w = e->define_stmt; + allocate(w); + // (_, special, v@{CollapseExpr,...}) -> (CollapseExpr.define_stmt.final, epsilon, v) + for (long f: compiled[w].fsa.finals) { + long g = stmt2offset[w]+f; + sorted_emplace(adj[g], epsilon, v); + if (g == i) + j++; + } + } + } + // remove (i, special, _) + adj[i].resize(j); + } + }; + allocate(stmt); + anno.fsa.adj = move(adj); + anno.assoc = move(assoc); + anno.deterministic = false; + DP(3, "# of states: %ld", anno.fsa.n()); + + // substring grammar & this nonterminal is not marked as intact + if (opt_substring_grammar && ! stmt->intact) { + DP(3, "Constructing substring grammar"); + anno.substring_grammar(); + DP(3, "# of states: %ld", anno.fsa.n()); + } + + vector> map0; + DP(3, "Determinize"); + anno.determinize(&starts, &map0); + vector sub_final2(anno.fsa.n()); + REP(i, anno.fsa.n()) + for (long u: map0[i]) { + if (sub_final[u]) + sub_final2[i] = true; + if (start2stmt.count(u)) { + DefineStmt* stmt = start2stmt[u]; + if (stmt2start[stmt] < 0) { + stmt->module->locfile.error_context(stmt->loc, "the start has been included in multiple DFA states"); + return false; + } + stmt2start[stmt] = ~ i; + } + } + sub_final = move(sub_final2); + start2stmt.clear(); + for (auto& it: stmt2start) { + it.second = ~ it.second; + start2stmt[it.second] = it.first; + } + DP(3, "# of states: %ld", anno.fsa.n()); + + DP(3, "Minimize"); + map0.clear(); + anno.minimize(&map0); + sub_final2.assign(anno.fsa.n(), false); + REP(i, anno.fsa.n()) + for (long u: map0[i]) { + if (sub_final[u]) + sub_final2[i] = true; + if (start2stmt.count(u)) { + DefineStmt* stmt = start2stmt[u]; + stmt2start[stmt] = i; + } + } + sub_final = move(sub_final2); + start2stmt.clear(); + for (auto& it: stmt2start) + start2stmt[it.second] = it.first; + DP(3, "# of states: %ld", anno.fsa.n()); + + if (! opt_keep_inaccessible) { + DP(3, "Keep accessible states"); + // roots: start, starts of DefineStmt associated to CallExpr + starts.clear(); + for (auto& it: stmt2start) + starts.push_back(it.second); + vector map1; + anno.accessible(&starts, map1); + sub_final2.assign(anno.fsa.n(), false); + REP(i, anno.fsa.n()) { + long u = map1[i]; + sub_final2[i] = sub_final[u]; + if (start2stmt.count(u)) + stmt2start[start2stmt[u]] = i; + } + sub_final = move(sub_final2); + start2stmt.clear(); + for (auto& it: stmt2start) + start2stmt[it.second] = it.first; + DP(3, "# of states: %ld", anno.fsa.n()); + + DP(3, "Keep co-accessible states"); + // roots: finals, finals of DefineStmt associated to CallExpr + map1.clear(); + anno.co_accessible(&sub_final, map1); + sub_final2.assign(anno.fsa.n(), false); + REP(i, anno.fsa.n()) { + long u = map1[i]; + sub_final2[i] = sub_final[u]; + if (start2stmt.count(u)) + stmt2start[start2stmt[u]] = i; + } + sub_final = move(sub_final2); + start2stmt.clear(); + for (auto& it: stmt2start) + start2stmt[it.second] = it.first; + DP(3, "# of states: %ld", anno.fsa.n()); + } + + stmt2final[stmt] = sub_final; + auto& call_addr = stmt2call_addr[stmt]; + call_addr.assign(anno.fsa.n(), make_pair(-1L, -1L)); + DP(3, "CallExpr"); + REP(i, anno.fsa.n()) + if (anno.fsa.has_call(i)) { + if (anno.fsa.adj[i].size() != 1 || anno.fsa.adj[i][0].first.second-anno.fsa.adj[i][0].first.first > 1) { + stmt->module->locfile.error_context(stmt->loc, "state %ld: CallExpr cannot coexist with other transitions", i); + for (auto it = anno.fsa.adj[i].begin(); it != anno.fsa.adj[i].end(); ) { + long from = it->first.first, to = it->first.second, v = it->second; + while (++it != anno.fsa.adj[i].end() && to == it->first.first && it->second == v) + to = it->first.second; + fprintf(stderr, " (%ld,%ld)\n", from, to-1); + } + return false; + } + for (auto aa: anno.assoc[i]) + if (has_start(aa.second)) + if (auto* e = dynamic_cast(aa.first)) // unique + call_addr[i] = {stmt2start[e->define_stmt], anno.fsa.adj[i][0].second}; + } + + DP(3, "Removing action/CallExpr labels"); + REP(i, anno.fsa.n()) { + long j = anno.fsa.adj[i].size(); + while (j && action_label_base < anno.fsa.adj[i][j-1].first.second) + if (anno.fsa.adj[i][j-1].first.first < action_label_base) + anno.fsa.adj[i][j-1].first.second = action_label_base; + else + j--; + anno.fsa.adj[i].resize(j); + } + + return true; +} + +//// Graphviz dot renderer + +void generate_graphviz(Module* mo) +{ + fprintf(output, "// Generated by 偃师, %s\n", mo->filename.c_str()); + for (Stmt* x = mo->toplevel; x; x = x->next) + if (auto stmt = dynamic_cast(x)) { + if (stmt->export_) { + FsaAnno& anno = compiled[stmt]; + + fprintf(output, "digraph \"%s\" {\n", mo->filename.c_str()); + bool start_is_final = false; + + // finals + indent(output, 1); + fprintf(output, "node[shape=doublecircle,color=olivedrab1,style=filled,fontname=Monospace];"); + for (long f: anno.fsa.finals) + if (f == anno.fsa.start) + start_is_final = true; + else + fprintf(output, " %ld", f); + fprintf(output, "\n"); + + // start + indent(output, 1); + if (start_is_final) + fprintf(output, "node[shape=doublecircle,color=orchid];"); + else + fprintf(output, "node[shape=circle,color=orchid];"); + fprintf(output, " %ld\n", anno.fsa.start); + + // other states + indent(output, 1); + fprintf(output, "node[shape=circle,color=black,style=\"\"]\n"); + + // edges + REP(u, anno.fsa.n()) { + unordered_map labels; + bool first = true; + auto it = anno.fsa.adj[u].begin(); + for (; it != anno.fsa.adj[u].end(); ++it) { + stringstream& lb = labels[it->second]; + if (! lb.str().empty()) + lb << ','; + if (it->first.first == it->first.second-1) + lb << it->first.first; + else + lb << it->first.first << '-' << it->first.second-1; + } + for (auto& lb: labels) { + indent(output, 1); + fprintf(output, "%ld -> %ld[label=\"%s\"]\n", u, lb.first, lb.second.str().c_str()); + } + } + } + } + fprintf(output, "}\n"); +} + +//// C++ renderer + +static void generate_final(const char* name, const vector& final) +{ + // comment + fprintf(output, " //static const long %sfinals[] = {", name); + bool first = true; + REP(i, final.size()) + if (final[i]) { + if (first) first = false; + else fprintf(output, ","); + fprintf(output, "%ld", i); + } + fprintf(output, "};\n"); + + first = true; + fprintf(output, " static const unsigned long %sfinal[] = {", name); + for (long j = 0, i = 0; i < final.size(); i += CHAR_BIT*sizeof(long)) { + ulong mask = 0; + for (; j < final.size() && j < i+CHAR_BIT*sizeof(long); j++) + if (final[j]) { + mask |= 1uL << (j-i); + } + if (i) fprintf(output, ","); + fprintf(output, "%#lx", mask); + } + fprintf(output, "};\n"); +} + +static void generate_cxx_export(DefineStmt* stmt) +{ + FsaAnno& anno = compiled[stmt]; + + // yanshi_%s_init + if (output_header) + fprintf(output_header, "extern long yanshi_%s_start;\n", stmt->lhs.c_str()); + fprintf(output, "long yanshi_%s_start = %ld;\n\n", stmt->lhs.c_str(), anno.fsa.start); + + // yanshi_%s_is_final + if (output_header) { + if (opt_gen_extern_c) fputs("extern \"C\" ", output_header); + fprintf(output_header, opt_gen_c ? +"bool yanshi_%s_is_final(const long* ret_stack, long ret_stack_len, long u);\n" +: +"bool yanshi_%s_is_final(const vector& ret_stack, long u);\n" +, stmt->lhs.c_str()); + } + if (opt_gen_extern_c) fputs("extern \"C\" ", output); + fprintf(output, opt_gen_c ? +"bool yanshi_%s_is_final(const long* ret_stack, long ret_stack_len, long u)\n" +: +"bool yanshi_%s_is_final(const vector& ret_stack, long u)\n" + , stmt->lhs.c_str()); + fprintf(output, "{\n"); + vector final(anno.fsa.n()); + for (long f: anno.fsa.finals) + final[f] = true; + generate_final("", final); + generate_final("sub_", stmt2final[stmt]); + fprintf(output, opt_gen_c ? +" for (long i = ret_stack_len; i; u = ret_stack[--i])\n" +: +" for (auto i = ret_stack.size(); i; u = ret_stack[--i])\n" +); + fprintf(output, +" if (! (0 <= u && u < %ld && sub_final[u/(CHAR_BIT*sizeof(long))] >> (u%%(CHAR_BIT*sizeof(long))) & 1))\n" +" return false;\n" +" return 0 <= u && u < %ld && final[u/(CHAR_BIT*sizeof(long))] >> (u%%(CHAR_BIT*sizeof(long))) & 1;\n" +"};\n\n" +, anno.fsa.n() , anno.fsa.n() +); + generate_transitions(stmt); +} + +void generate_cxx(Module* mo) +{ + fprintf(output, "// Generated by 偃师, %s\n", mo->filename.c_str()); + fprintf(output, "#include \n"); + if (! opt_gen_c) { + fprintf(output, "#include \n"); + fprintf(output, "using namespace std;\n"); + } else { + fprintf(output, "#include \n"); + } + if (opt_standalone) { + fputs( +"#include \n" +"#include \n" +"#include \n" +"#include \n" +"#include \n" +"#include \n" +"#include \n" +"#include \n" +"#include \n" +"#include \n" +"#include \n" +"using namespace std;\n" +, output); + } + if (output_header) { + fputs("#pragma once\n", output_header); + if (! opt_gen_c) { + fprintf(output_header, "#include \n"); + fprintf(output_header, "using std::vector;\n"); + } else { + fprintf(output_header, "#include \n"); + } + } + fprintf(output, "\n"); + DefineStmt* main_export = NULL; + for (Stmt* x = mo->toplevel; x; x = x->next) + if (auto xx = dynamic_cast(x)) { + if (xx->export_) { + if (! main_export) + main_export = xx; + generate_cxx_export(xx); + } + } else if (auto xx = dynamic_cast(x)) + fprintf(output, "%s", xx->code.c_str()); + if (opt_standalone && main_export) { + fprintf(output, +"\n" +"int main(int argc, char* argv[])\n" +"{\n" +" setlocale(LC_ALL, \"\");\n" +" string utf8;\n" +" const char* p;\n" +" long c, u = yanshi_%s_start, pref = 0;\n" +, main_export->lhs.c_str()); + if (opt_gen_c) + fprintf(output, " long ret_stack[%ld], ret_stack_len = 0;\n", opt_max_return_stack); + else + fprintf(output, " vector ret_stack;\n"); + fprintf(output, +" if (argc == 2)\n" +" utf8 = argv[1];\n" +" else {\n" +" FILE* f = argc == 1 ? stdin : fopen(argv[1], \"r\");\n" +" while ((c = fgetc(f)) != EOF)\n" +" utf8 += c;\n" +" fclose(f);\n" +" }\n" +" u32string utf32 = wstring_convert, char32_t>{}.from_bytes(utf8);\n" +); + fprintf(output, opt_gen_c ? +" printf(\"\\033[%%s33m%%ld \\033[m\", yanshi_%s_is_final(ret_stack, ret_stack_len, u) ? \"1;\" : \"\", u);\n" +: +" printf(\"\\033[%%s33m%%ld \\033[m\", yanshi_%s_is_final(ret_stack, u) ? \"1;\" : \"\", u);\n" +, main_export->lhs.c_str() +); + fprintf(output, +" for (char32_t c: utf32) {\n"); + fprintf(output, opt_gen_c ? +" u = yanshi_%s_transit(ret_stack, &ret_stack_len, u, c);\n" +: +" u = yanshi_%s_transit(ret_stack, u, c);\n" +, main_export->lhs.c_str()); + fprintf(output, +" if (c > WCHAR_MAX || iswcntrl(c)) printf(\"%%\" PRIuLEAST32 \" \", c);\n" +" else cout << wstring_convert, char32_t>{}.to_bytes(c) << ' ';\n"); + fprintf(output, opt_gen_c ? +" printf(\"\\033[%%s33m%%ld \\033[m\", yanshi_%s_is_final(ret_stack, ret_stack_len, u) ? \"1;\" : \"\", u);\n" +: +" printf(\"\\033[%%s33m%%ld \\033[m\", yanshi_%s_is_final(ret_stack, u) ? \"1;\" : \"\", u);\n" +, main_export->lhs.c_str()); + fprintf(output, +" if (u < 0) break;\n" +" pref++;\n" +" }\n"); + fprintf(output, opt_gen_c ? +" printf(\"\\nlen: %%zd\\npref: %%ld\\nstate: %%ld\\nfinal: %%s\\n\", utf32.size(), pref, u, yanshi_%s_is_final(ret_stack, ret_stack_len, u) ? \"true\" : \"false\");\n" +"}\n" +: +" printf(\"\\nlen: %%zd\\npref: %%ld\\nstate: %%ld\\nfinal: %%s\\n\", utf32.size(), pref, u, yanshi_%s_is_final(ret_stack, u) ? \"true\" : \"false\");\n" +"}\n" +, main_export->lhs.c_str()); + } +} diff --git a/yanshi/src/compiler.hh b/yanshi/src/compiler.hh new file mode 100644 index 0000000..f80b165 --- /dev/null +++ b/yanshi/src/compiler.hh @@ -0,0 +1,14 @@ +#pragma once +#include "fsa_anno.hh" +#include "syntax.hh" + +#include +using std::unordered_map; + +void print_assoc(const FsaAnno& anno); +void print_automaton(const Fsa& fsa); +void compile(DefineStmt*); +bool compile_export(DefineStmt* stmt); +void generate_cxx(Module* mo); +void generate_graphviz(Module* mo); +extern unordered_map compiled; diff --git a/yanshi/src/fsa.cc b/yanshi/src/fsa.cc new file mode 100644 index 0000000..42c3e0d --- /dev/null +++ b/yanshi/src/fsa.cc @@ -0,0 +1,571 @@ +#include "common.hh" +#include "fsa.hh" +#include "option.hh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +using namespace std; + +namespace std +{ + template + struct hash> { + size_t operator()(const vector& v) const { + hash h; + size_t r = 0; + for (auto x: v) + r = r*17+h(x); + return r; + } + }; +} + +void Fsa::check() const +{ + REP(i, n()) + FOR(j, 1, adj[i].size()) + assert(adj[i][j-1].first.second == 0 && + adj[i][j].first.second == 0 || + adj[i][j-1].first.second <= adj[i][j].first.first); +} + +bool Fsa::has(long u, long c) const +{ + auto it = upper_bound(ALL(adj[u]), make_pair(make_pair(c, LONG_MAX), LONG_MAX)); + return it != adj[u].begin() && c < (--it)->first.second; +} + +bool Fsa::has_call(long u) const +{ + auto it = upper_bound(ALL(adj[u]), make_pair(make_pair(call_label_base, LONG_MAX), LONG_MAX)); + return (it != adj[u].end() && it->first.first < call_label) || (it != adj[u].begin() && call_label_base < (--it)->first.second); +} + +bool Fsa::has_call_or_collapse(long u) const +{ + auto it = upper_bound(ALL(adj[u]), make_pair(make_pair(call_label_base, LONG_MAX), LONG_MAX)); + return it != adj[u].end() || (it != adj[u].begin() && call_label_base < (--it)->first.second); +} + +long Fsa::transit(long u, long c) const +{ + auto it = upper_bound(ALL(adj[u]), make_pair(make_pair(c, LONG_MAX), LONG_MAX)); + return it != adj[u].begin() && c < (--it)->first.second ? it->second : -1; +} + +bool Fsa::is_final(long x) const +{ + return binary_search(ALL(finals), x); +} + +void Fsa::epsilon_closure(vector& src) const +{ + static vector vis; + if (n() > vis.size()) + vis.resize(n()); + for (long i: src) + vis[i] = true; + REP(i, src.size()) { + long u = src[i]; + for (auto& e: adj[u]) { + if (-1 < e.first.first) break; + if (! vis[e.second]) { + vis[e.second] = true; + src.push_back(e.second); + } + } + } + for (long i: src) + vis[i] = false; + sort(ALL(src)); +} + +Fsa Fsa::operator~() const +{ + long accept = n(); + Fsa r; + r.start = start; + r.adj.resize(accept+1); + REP(i, accept) { + long j = 0; + for (auto& e: adj[i]) { + if (j < e.first.first) + r.adj[i].emplace_back(make_pair(j, e.first.first), accept); + r.adj[i].emplace_back(e.first, e.second); + j = e.first.second; + } + if (j < AB) + r.adj[i].emplace_back(make_pair(j, AB), accept); + } + r.adj[accept].emplace_back(make_pair(0, AB), accept); + vector new_finals; + auto j = finals.begin(); + REP(i, accept+1) { + while (j != finals.end() && *j < i) + ++j; + if (j == finals.end() || *j != i) + new_finals.push_back(i); + } + r.finals = move(new_finals); + return r; +} + +void Fsa::accessible(const vector* starts, function relate) +{ + vector q{start}, id(n(), 0); + id[start] = 1; + if (starts) + for (long u: *starts) + if (! id[u]) { + id[u] = 1; + q.push_back(u); + } + REP(i, q.size()) { + long u = q[i]; + for (auto& e: adj[u]) { + //if (e.first.first >= AB) break; + if (! id[e.second]) { + id[e.second] = 1; + q.push_back(e.second); + } + } + } + + long j = 0; + REP(i, n()) + id[i] = id[i] ? j++ : -1; + + auto it = finals.begin(), it2 = it; + REP(i, n()) + if (id[i] >= 0) { + relate(i); + if (start == i) + start = id[i]; + while (it != finals.end() && *it < i) + ++it; + if (it != finals.end() && *it == i) + *it2++ = id[i]; + long k = 0; + for (auto& e: adj[i]) + if (id[e.second] >= 0) + adj[i][k++] = {e.first, id[e.second]}; // unordered unless deterministic + adj[i].resize(k); + if (id[i] != i) + adj[id[i]] = move(adj[i]); + } + finals.erase(it2, finals.end()); + adj.resize(j); +} + +void Fsa::co_accessible(const vector*final, function relate) +{ + vector> radj(n()); + REP(i, n()) + for (auto& e: adj[i]) { + //if (e.first.first >= AB) break; + radj[e.second].push_back(i); + } + REP(i, n()) + sort(ALL(radj[i])); + vector q = finals, id(n(), 0); + for (long f: finals) + id[f] = 1; + if (final) + REP(i, n()) + if ((*final)[i] && ! id[i]) { + id[i] = 1; + q.push_back(i); + } + REP(i, q.size()) { + long u = q[i]; + for (auto& v: radj[u]) + if (! id[v]) { + id[v] = 1; + q.push_back(v); + } + } + if (! id[start]) { + start = 0; + finals.clear(); + adj.assign(1, {}); + return; + } + + long j = 0; + REP(i, n()) + id[i] = id[i] ? j++ : -1; + + auto it = finals.begin(), it2 = it; + REP(i, n()) + if (id[i] >= 0) { + relate(i); + if (start == i) + start = id[i]; + while (it != finals.end() && *it < i) + ++it; + if (it != finals.end() && *it == i) + *it2++ = id[i]; + long k = 0; + for (auto& e: adj[i]) + if (id[e.second] >= 0) + adj[i][k++] = {e.first, id[e.second]}; // unordered unless deterministic + adj[i].resize(k); + if (id[i] != i) + adj[id[i]] = move(adj[i]); + } + finals.erase(it2, finals.end()); + adj.resize(j); +} + +Fsa Fsa::difference(const Fsa& rhs, function relate) const +{ + Fsa r; + vector> q; + unordered_map m; + q.emplace_back(start, rhs.start); + m[(rhs.n()+1) * start + rhs.start] = 0; + r.start = 0; + REP(i, q.size()) { + long u0, u1, v0, v1; + tie(u0, u1) = q[i]; + if (is_final(u0) && ! rhs.is_final(u1)) + r.finals.push_back(i); + r.adj.emplace_back(); + relate(u0); + vector::const_iterator it0 = adj[u0].begin(), it1, it1e; + if (u1 == rhs.n()) + it1 = it1e = rhs.adj[0].end(); + else { + it1 = rhs.adj[u1].begin(); + it1e = rhs.adj[u1].end(); + } + long last = LONG_MIN; + while (it0 != adj[u0].end()) { + long from = max(last, it0->first.first), to = it0->first.second; + while (it1 != it1e && it1->first.second <= from) + ++it1; + if (it1 != it1e) + to = min(to, from < it1->first.first ? it1->first.first : it1->first.second); + last = to; + long v1 = it1 != it1e && it1->first.first <= from ? it1->second : rhs.n(), + t = (rhs.n()+1) * it0->second + v1; + auto mit = m.find(t); + if (mit == m.end()) { + mit = m.emplace(t, m.size()).first; + q.emplace_back(it0->second, v1); + } + r.adj[i].emplace_back(make_pair(from, to), mit->second); + if (to == it0->first.second) + ++it0; + } + } + return r; +} + +Fsa Fsa::intersect(const Fsa& rhs, function relate) const +{ + Fsa r; + vector> q; + long u0, u1, v0, v1; + unordered_map m; + q.emplace_back(start, rhs.start); + m[rhs.n() * start + rhs.start] = 0; + r.start = 0; + REP(i, q.size()) { + tie(u0, u1) = q[i]; + if (is_final(u0) && rhs.is_final(u1)) + r.finals.push_back(i); + r.adj.emplace_back(); + relate(u0, u1); + auto it0 = adj[u0].begin(), it1 = rhs.adj[u1].begin(); + while (it0 != adj[u0].end() && it1 != rhs.adj[u1].end()) { + if (it0->first.second <= it1->first.first) + ++it0; + else if (it1->first.second <= it0->first.first) + ++it1; + else { + long t = rhs.n() * it0->second + it1->second; + auto mit = m.find(t); + if (mit == m.end()) { + mit = m.emplace(t, m.size()).first; + q.emplace_back(it0->second, it1->second); + } + r.adj[i].emplace_back(make_pair(max(it0->first.first, it1->first.first), min(it0->first.second, it1->first.second)), mit->second); + if (it0->first.second < it1->first.second) + ++it0; + else if (it0->first.second > it1->first.second) + ++it1; + else + ++it0, ++it1; + } + } + } + return r; +} + +Fsa Fsa::determinize(const vector* starts, function&)> relate) const +{ + Fsa r; + r.start = 0; + unordered_map, long> m; + vector::const_iterator> its(n()); + vector vs{start}; + vector> events; + stack> st; + epsilon_closure(vs); + m[vs] = 0; + st.push(move(vs)); + if (starts) + for (long u: *starts) { + vs.assign(1, u); + epsilon_closure(vs); + if (! m.count(vs)) { + m.emplace(vs, m.size()); + st.push(move(vs)); + } + } + while (st.size()) { + vector x = move(st.top()); + st.pop(); + long id = m[x]; + if (id+1 > r.adj.size()) + r.adj.resize(id+1); + relate(id, x); + bool final = false; + events.clear(); + for (long u: x) { + if (is_final(u)) + final = true; + for (auto& e: adj[u]) { + events.emplace_back(e.first.first, e.second); + events.emplace_back(e.first.second, ~ e.second); + } + } + if (final) + r.finals.push_back(id); + long last = 0; + multiset live; + sort(ALL(events)); + for (auto& ev: events) { + if (last < ev.first) { + if (live.size()) { + vs.assign(ALL(live)); + vs.erase(unique(ALL(vs)), vs.end()); + epsilon_closure(vs); + auto mit = m.find(vs); + if (mit == m.end()) { + mit = m.emplace(vs, m.size()).first; + st.push(vs); + } + if (r.adj[id].size() && r.adj[id].back().first.second == last && r.adj[id].back().second == mit->second) // coalesce two edges + r.adj[id].back().first.second = ev.first; + else + r.adj[id].emplace_back(make_pair(last, ev.first), mit->second); + } + last = ev.first; + } + if (ev.second >= 0) + live.insert(ev.second); + else + live.erase(live.find(~ ev.second)); + } + } + sort(ALL(r.finals)); + return r; +} + +Fsa Fsa::distinguish(function&)> relate) const +{ + vector scale; + REP(i, n()) + for (auto& e: adj[i]) { + scale.push_back(e.first.first); + scale.push_back(e.first.second); + } + sort(ALL(scale)); + scale.erase(unique(ALL(scale)), scale.end()); + + vector>> radj(n()); + REP(i, n()) + for (auto& e: adj[i]) { + long from = lower_bound(ALL(scale), e.first.first) - scale.begin(), + to = lower_bound(ALL(scale), e.first.second) - scale.begin(); + FOR(j, from, to) + radj[e.second].emplace_back(j, i); + } + REP(i, n()) + sort(ALL(radj[i])); + vector L(n()), R(n()), B(n()), C(n(), 0), CC(n(), 0); + vector mark(n(), false); + + // distinguish finals & non-finals + long fx = -1, x = -1, fy = -1, y = -1, j = 0; + REP(i, n()) + if (j < finals.size() && finals[j] == i) { + j++; + if (y < 0) + fy = i; + else + R[y] = i; + C[B[i] = fy]++; + L[i] = y; + y = i; + } else { + if (x < 0) + fx = i; + else + R[x] = i; + C[B[i] = fx]++; + L[i] = x; + x = i; + } + if (x >= 0) + L[fx] = x, R[x] = fx; + if (y >= 0) + L[fy] = y, R[y] = fy; + + set> refines; + auto labels = [&](long fx) { + vector lb; + for (long x = fx; ; ) { + for (auto& e: radj[x]) + lb.push_back(e.first); + if ((x = R[x]) == fx) break; + } + sort(ALL(lb)); + lb.erase(unique(ALL(lb)), lb.end()); + return lb; + }; + + if (fx >= 0) + for (long a: labels(fx)) + refines.emplace(a, fx); + if (fy >= 0) + for (long a: labels(fy)) + refines.emplace(a, fy); + while (refines.size()) { + long a; + tie(a, fx) = *refines.begin(); + refines.erase(refines.begin()); + // count + vector bs; + for (x = fx; ; ) { + auto it = lower_bound(ALL(radj[x]), make_pair(a, 0L)), + ite = upper_bound(ALL(radj[x]), make_pair(a, n())); + for (; it != ite; ++it) { + y = it->second; + if (! CC[B[y]]++) + bs.push_back(B[y]); + mark[y] = true; + } + if ((x = R[x]) == fx) break; + } + // for each refinable set + for (long fy: bs) { + if (CC[fy] < C[fy]) { + long fu = -1, u = -1, cu = 0, + fv = -1, v = -1, cv = 0; + vector lb = labels(fy); + for (long i = fy; ; ) { + if (mark[i]) { + mark[i] = false; + if (u < 0) + C[fu = i] = 0; + else + R[u] = i; + C[fu]++; + B[i] = fu; + L[i] = u; + u = i; + } else { + if (v < 0) + C[fv = i] = 0; + else + R[v] = i; + C[fv]++; + B[i] = fv; + L[i] = v; + v = i; + } + if ((i = R[i]) == fy) break; + } + L[fu] = u, R[u] = fu; + L[fv] = v, R[v] = fv; + //REP(a, AB+1) + for (long a: lb) + if (refines.count({a, fy})) + refines.emplace(a, fu != fy ? fu : fv); + else + refines.emplace(a, C[fu] < C[fv] ? fu : fv); + } else + for (long i = fy; ; ) { + mark[i] = false; + if ((i = R[i]) == fy) break; + } + CC[fy] = 0; + } + // clear marks + for (x = fx; ; ) { + auto it = lower_bound(ALL(radj[x]), make_pair(a, 0L)), + ite = upper_bound(ALL(radj[x]), make_pair(a, n())); + for (; it != ite; ++it) { + y = it->second; + CC[B[y]] = 0; + mark[y] = false; + } + if ((x = R[x]) == fx) break; + } + } + + Fsa r; + long nn = 0; + vector vs; + REP(i, n()) + if (B[i] == i) { + vs.clear(); + for (long j = i; ; ) { + B[j] = nn; + vs.push_back(j); + if ((j = R[j]) == i) break; + } + relate(vs); + if (binary_search(ALL(finals), i)) + r.finals.push_back(nn); + nn++; + } + r.start = B[start]; + r.adj.resize(nn); + REP(i, n()) + for (auto& e: adj[i]) + r.adj[B[i]].emplace_back(e.first, B[e.second]); + REP(i, nn) { + // merge edges with the same destination + sort(ALL(r.adj[i]), [](const Edge& x, const Edge& y) { + return x.second != y.second ? x.second < y.second : x.first < y.first; + }); + auto it2 = r.adj[i].begin(); + for (auto it = r.adj[i].begin(); it != r.adj[i].end(); ) { + long v = it->second, from = it->first.first, to = it->first.second; + while (++it != r.adj[i].end() && it->second == v) + if (it->first.first <= to) + to = max(to, it->first.second); + else { + *it2++ = make_pair(make_pair(from, to), v); + tie(from, to) = it->first; + } + *it2++ = make_pair(make_pair(from, to), v); + } + r.adj[i].erase(it2, r.adj[i].end()); + sort(ALL(r.adj[i])); + } + return r; +} diff --git a/yanshi/src/fsa.hh b/yanshi/src/fsa.hh new file mode 100644 index 0000000..1a5ed83 --- /dev/null +++ b/yanshi/src/fsa.hh @@ -0,0 +1,40 @@ +#pragma once +#include +#include +#include +using std::function; +using std::pair; +using std::vector; + +typedef pair Label; +typedef pair Edge; + +const Label epsilon{-1L, 0L}; + +struct Fsa { + long start; + vector finals; // sorted + vector> adj; // sorted + + void check() const; + long n() const { return adj.size(); } + bool is_final(long x) const; + bool has(long u, long c) const; + bool has_call(long u) const; + bool has_call_or_collapse(long u) const; + long transit(long u, long c) const; + void epsilon_closure(vector& src) const; + Fsa operator~() const; + // a -> a + void accessible(const vector* starts, function relate); + // a -> a + void co_accessible(const vector* final, function relate); + // DFA -> DFA -> DFA + Fsa intersect(const Fsa& rhs, function relate) const; + // DFA -> DFA -> DFA + Fsa difference(const Fsa& rhs, function relate) const; + // DFA -> DFA + Fsa distinguish(function&)> relate) const; + // * -> DFA + Fsa determinize(const vector* starts, function&)> relate) const; +}; diff --git a/yanshi/src/fsa_anno.cc b/yanshi/src/fsa_anno.cc new file mode 100644 index 0000000..3e6c673 --- /dev/null +++ b/yanshi/src/fsa_anno.cc @@ -0,0 +1,476 @@ +#include "common.hh" +#include "compiler.hh" +#include "fsa_anno.hh" +#include "loader.hh" +#include "option.hh" + +#include +#include +#include +#include +#include +using namespace std; + +bool operator<(ExprTag x, ExprTag y) +{ + return long(x) < long(y); +} + +bool assoc_has_expr(vector>& as, Expr* x) +{ + auto it = lower_bound(ALL(as), make_pair(x, ExprTag(0))); + return it != as.end() && it->first == x; +} + +void sort_assoc(vector>& as) +{ + sort(ALL(as)); + auto i = as.begin(), j = i, k = i; + for (; i != as.end(); i = j) { + while (++j != as.end() && i->first == j->first) + i->second = ExprTag(long(i->second) | long(j->second)); + *k++ = *i; + } + as.erase(k, as.end()); +} + +void FsaAnno::add_assoc(Expr& expr) +{ + // has actions: actions need tags to differentiate 'entering', 'leaving', ... + // 'intact': states with the 'inner' tag cannot be connected to start/final in substring grammar + // 'CallExpr' 'CollapseExpr': differentiate states representing 'CallExpr' 'CollapseExpr' (u, special, v) + // 'opt_mode': displaying possible positions for given strings in interactive mode + if (expr.no_action() && ! expr.stmt->intact && ! dynamic_cast(&expr) && ! dynamic_cast(&expr) && opt_mode != Mode::interactive) + return; + auto j = fsa.finals.begin(); + REP(i, fsa.n()) { + ExprTag tag = ExprTag(0); + if (i == fsa.start) + tag = ExprTag::start; + while (j != fsa.finals.end() && *j < i) + ++j; + if (j != fsa.finals.end() && *j == i) + tag = ExprTag(long(tag) | long(ExprTag::final)); + if (tag == ExprTag(0)) + tag = ExprTag::inner; + sorted_insert(assoc[i], make_pair(&expr, tag)); + } + // Add pseudo transitions with labels [ACTION_LABEL_BASE, COLLAPSE_LABEL_BASE) to prevent its merge with other states + if (expr.leaving.size() || expr.entering.size() || expr.transiting.size()) + for (auto action: expr.transiting) + REP(i, fsa.n()) { + fsa.adj[i].emplace_back(make_pair(action_label, action_label+1), i); + action_label++; + } + else if (expr.finishing.size()) + for (long f: fsa.finals) { + fsa.adj[f].emplace_back(make_pair(action_label, action_label+1), f); + action_label++; + } +} + +void FsaAnno::accessible(const vector* starts, vector& mapping) { + long allo = 0; + auto relate = [&](long x) { + if (allo != x) + assoc[allo] = move(assoc[x]); + allo++; + mapping.push_back(x); + }; + fsa.accessible(starts, relate); + assoc.resize(allo); +} + +void FsaAnno::co_accessible(const vector* final, vector& mapping) { + long allo = 0; + auto relate = [&](long x) { + if (allo != x) + assoc[allo] = move(assoc[x]); + allo++; + mapping.push_back(x); + }; + fsa.co_accessible(final, relate); + if (fsa.finals.empty()) { // 'start' does not produce acceptable strings + assoc.assign(1, {}); + mapping.assign(1, 0); + deterministic = true; + return; + } + if (! deterministic) + REP(i, fsa.n()) + sort(ALL(fsa.adj[i])); + assoc.resize(allo); +} + +void FsaAnno::complement(ComplementExpr* expr) { + if (! deterministic) + fsa = fsa.determinize(NULL, [&](long, const vector&){}); + fsa = ~ fsa; + assoc.assign(fsa.n(), {}); + deterministic = true; +} + +void FsaAnno::concat(FsaAnno& rhs, ConcatExpr* expr) { + long ln = fsa.n(), rn = rhs.fsa.n(); + for (long f: fsa.finals) + emplace_front(fsa.adj[f], epsilon, ln+rhs.fsa.start); + for (auto& es: rhs.fsa.adj) { + for (auto& e: es) + e.second += ln; + fsa.adj.emplace_back(move(es)); + } + fsa.finals = move(rhs.fsa.finals); + for (long& f: fsa.finals) + f += ln; + assoc.resize(fsa.n()); + REP(i, rhs.fsa.n()) + assoc[ln+i] = move(rhs.assoc[i]); + if (expr) + add_assoc(*expr); + deterministic = false; +} + +void FsaAnno::determinize(const vector* starts, vector>* mapping) { + if (deterministic) + return; + decltype(assoc) new_assoc; + auto relate = [&](long id, const vector& xs) { + if (id+1 > new_assoc.size()) { + new_assoc.resize(id+1); + if (mapping) + mapping->resize(id+1); + } + auto& as = new_assoc[id]; + for (long x: xs) + as.insert(as.end(), ALL(assoc[x])); + sort_assoc(as); + if (mapping) + (*mapping)[id] = xs; + }; + fsa = fsa.determinize(starts, relate); + assoc = move(new_assoc); + deterministic = true; +} + +void FsaAnno::difference(FsaAnno& rhs, DifferenceExpr* expr) { + vector> rel0; + decltype(rhs.assoc) new_assoc; + auto relate0 = [&](long id, const vector& xs) { + if (id+1 > rel0.size()) + rel0.resize(id+1); + rel0[id] = xs; + }; + auto relate = [&](long x) { + if (rel0.empty()) + new_assoc.emplace_back(assoc[x]); + else { + new_assoc.emplace_back(); + auto& as = new_assoc.back(); + for (long u: rel0[x]) + as.insert(as.end(), ALL(assoc[u])); + sort_assoc(as); + } + }; + if (! deterministic) + fsa = fsa.determinize(NULL, relate0); + if (! rhs.deterministic) + rhs.fsa = rhs.fsa.determinize(NULL, [](long, const vector&) {}); + fsa = fsa.difference(rhs.fsa, relate); + assoc = move(new_assoc); + if (expr) + add_assoc(*expr); + deterministic = true; +} + +FsaAnno FsaAnno::epsilon_fsa(EpsilonExpr* expr) { + FsaAnno r; + r.fsa.start = 0; + r.fsa.finals.push_back(0); + r.fsa.adj.resize(1); + r.assoc.resize(1); + if (expr) + r.add_assoc(*expr); + r.deterministic = true; + return r; +} + +void FsaAnno::intersect(FsaAnno& rhs, IntersectExpr* expr) { + decltype(rhs.assoc) new_assoc; + vector> rel0, rel1; + auto relate0 = [&](long id, const vector& xs) { + if (id+1 > rel0.size()) + rel0.resize(id+1); + rel0[id] = xs; + }; + auto relate1 = [&](long id, const vector& xs) { + if (id+1 > rel1.size()) + rel1.resize(id+1); + rel1[id] = xs; + }; + auto relate = [&](long x, long y) { + new_assoc.emplace_back(); + auto& as = new_assoc.back(); + if (rel0.empty()) + as.insert(as.end(), ALL(assoc[x])); + else + for (long u: rel0[x]) + as.insert(as.end(), ALL(assoc[u])); + if (rel1.empty()) + as.insert(as.end(), ALL(rhs.assoc[y])); + else + for (long v: rel1[y]) + as.insert(as.end(), ALL(rhs.assoc[v])); + sort_assoc(as); + }; + if (! deterministic) + fsa = fsa.determinize(NULL, relate0); + if (! rhs.deterministic) + rhs.fsa = rhs.fsa.determinize(NULL, relate1); + fsa = fsa.intersect(rhs.fsa, relate); + assoc = move(new_assoc); + if (expr) + add_assoc(*expr); + deterministic = true; +} + +void FsaAnno::minimize(vector>* mapping) { + assert(deterministic); + decltype(assoc) new_assoc; + auto relate = [&](vector& xs) { + new_assoc.emplace_back(); + auto& as = new_assoc.back(); + for (long x: xs) + as.insert(as.end(), ALL(assoc[x])); + sort_assoc(as); + if (mapping) + mapping->push_back(xs); + }; + fsa = fsa.distinguish(relate); + assoc = move(new_assoc); +} + +void FsaAnno::union_(FsaAnno& rhs, UnionExpr* expr) { + long ln = fsa.n(), rn = rhs.fsa.n(), src = ln+rn, + old_lsrc = fsa.start; + fsa.start = src; + for (long f: rhs.fsa.finals) + fsa.finals.push_back(ln+f); + for (auto& es: rhs.fsa.adj) { + for (auto& e: es) + e.second += ln; + fsa.adj.emplace_back(move(es)); + } + fsa.adj.emplace_back(); + fsa.adj[src].emplace_back(epsilon, old_lsrc); + fsa.adj[src].emplace_back(epsilon, ln+rhs.fsa.start); + assoc.resize(fsa.n()); + REP(i, rhs.fsa.n()) + assoc[ln+i] = move(rhs.assoc[i]); + if (expr) + add_assoc(*expr); + deterministic = false; +} + +void FsaAnno::plus(PlusExpr* expr) { + for (long f: fsa.finals) + emplace_front(fsa.adj[f], epsilon, fsa.start); + if (expr) + add_assoc(*expr); + deterministic = false; +} + +void FsaAnno::question(QuestionExpr* expr) { + long src = fsa.n(), sink = src+1, old_src = fsa.start; + fsa.start = src; + fsa.adj.emplace_back(); + fsa.adj.emplace_back(); + fsa.adj[src].emplace_back(epsilon, old_src); + fsa.adj[src].emplace_back(epsilon, sink); + fsa.finals.push_back(sink); + assoc.resize(fsa.n()); + if (expr) + add_assoc(*expr); + deterministic = false; +} + +void FsaAnno::repeat(RepeatExpr& expr) { + FsaAnno r = epsilon_fsa(NULL); + REP(i, expr.low) { + FsaAnno t = *this; + r.concat(t, NULL); + } + if (expr.high == LONG_MAX) { + star(NULL); + r.concat(*this, NULL); + } else if (expr.low < expr.high) { + FsaAnno rhs = epsilon_fsa(NULL), x = *this; + ROF(i, 0, expr.high-expr.low) { + FsaAnno t = x; + rhs.union_(t, NULL); + if (i) { + t = *this; + x.concat(t, NULL); + } + } + r.concat(rhs, NULL); + } + r.deterministic = false; + *this = move(r); +} + +void FsaAnno::star(StarExpr* expr) { + long src = fsa.n(), sink = src+1, old_src = fsa.start; + fsa.start = src; + fsa.adj.emplace_back(); + fsa.adj.emplace_back(); + fsa.adj[src].emplace_back(epsilon, old_src); + fsa.adj[src].emplace_back(epsilon, sink); + for (long f: fsa.finals) { + sorted_emplace(fsa.adj[f], epsilon, old_src); + sorted_emplace(fsa.adj[f], epsilon, sink); + } + fsa.finals.assign(1, sink); + assoc.resize(fsa.n()); + if (expr) + add_assoc(*expr); + deterministic = false; +} + +FsaAnno FsaAnno::bracket(BracketExpr& expr) { + FsaAnno r; + r.fsa.start = 0; + r.fsa.finals = {1}; + r.fsa.adj.resize(2); + for (auto& x: expr.intervals.to) + r.fsa.adj[0].emplace_back(x, 1); + r.assoc.resize(2); + r.add_assoc(expr); + r.deterministic = true; + return r; +} + +FsaAnno FsaAnno::call(CallExpr& expr) { + // represented by (0, special, 1) + FsaAnno r; + r.fsa.start = 0; + r.fsa.finals = {1}; + r.fsa.adj.resize(2); + r.fsa.adj[0].emplace_back(make_pair(call_label, call_label+1), 1); + call_label++; + r.assoc.resize(2); + r.add_assoc(expr); + r.deterministic = true; + return r; +} + +FsaAnno FsaAnno::collapse(CollapseExpr& expr) { + // represented by (0, special, 1) + FsaAnno r; + r.fsa.start = 0; + r.fsa.finals = {1}; + r.fsa.adj.resize(2); + r.fsa.adj[0].emplace_back(make_pair(collapse_label, collapse_label+1), 1); + collapse_label++; + r.assoc.resize(2); + r.add_assoc(expr); + r.deterministic = true; + return r; +} + +FsaAnno FsaAnno::dot(DotExpr* expr) { + FsaAnno r; + r.fsa.start = 0; + r.fsa.finals = {1}; + r.fsa.adj.resize(2); + r.fsa.adj[0].emplace_back(make_pair(0L, AB), 1); + r.assoc.resize(2); + if (expr) + r.add_assoc(*expr); + r.deterministic = true; + return r; +} + +FsaAnno FsaAnno::embed(EmbedExpr& expr) { + if (expr.define_stmt) { + FsaAnno r = compiled[expr.define_stmt]; + // change the labels to differentiate instances of CallExpr + REP(i, r.fsa.n()) { + auto it = upper_bound(ALL(r.fsa.adj[i]), make_pair(make_pair(call_label_base, LONG_MAX), LONG_MAX)); + if (it != r.fsa.adj[i].begin() && call_label_base < (it-1)->first.second) + --it; + for (; it != r.fsa.adj[i].end() && it->first.first < call_label; ++it) { + assert(call_label_base <= it->first.first); + long t = it->first.second-it->first.first; + it->first.first = call_label; + call_label += t; + it->first.second = call_label; + assert(it->first.second <= call_label); + } + } + r.add_assoc(expr); + return r; + } else { // macro + FsaAnno r; + r.fsa.start = 0; + r.fsa.finals = {1}; + r.fsa.adj.resize(2); + r.fsa.adj[0].emplace_back(make_pair(expr.macro_value, expr.macro_value+1), 1); + r.assoc.resize(2); + r.add_assoc(expr); + r.deterministic = true; + return r; + } +} + +FsaAnno FsaAnno::literal(LiteralExpr& expr) { + FsaAnno r; + r.fsa.start = 0; + long len = 0; + if (opt_bytes) { + len = expr.literal.size(); + r.fsa.adj.resize(len+1); + REP(i, expr.literal.size()) { + long c = (u8)expr.literal[i]; + r.fsa.adj[i].emplace_back(make_pair(c, c+1), i+1); + } + } else { + for (i32 c, i = 0; i < expr.literal.size(); len++) { + U8_NEXT_OR_FFFD(expr.literal.c_str(), i, expr.literal.size(), c); + r.fsa.adj.emplace_back(); + r.fsa.adj[len].emplace_back(make_pair(c, c+1), len+1); + } + r.fsa.adj.emplace_back(); + } + r.fsa.finals.push_back(len); + r.assoc.resize(len+1); + r.add_assoc(expr); + r.deterministic = true; + return r; +} + +void FsaAnno::substring_grammar() { + long src = fsa.n(), sink = src+1, old_src = fsa.start; + fsa.start = src; + fsa.adj.emplace_back(); + fsa.adj.emplace_back(); + REP(i, src) { + bool ok = true; + for (auto aa: assoc[i]) + if (auto e = dynamic_cast(aa.first)) { + if (e->define_stmt->intact && has_inner(aa.second)) { + ok = false; + break; + } + } else if (aa.first->stmt->intact && has_inner(aa.second)) { + ok = false; + break; + } + if (ok || i == old_src) + fsa.adj[src].emplace_back(epsilon, i); + if (ok || fsa.is_final(i)) + emplace_front(fsa.adj[i], epsilon, sink); + } + fsa.finals.assign(1, sink); + assoc.resize(fsa.n()); + deterministic = false; +} diff --git a/yanshi/src/fsa_anno.hh b/yanshi/src/fsa_anno.hh new file mode 100644 index 0000000..0e6d484 --- /dev/null +++ b/yanshi/src/fsa_anno.hh @@ -0,0 +1,44 @@ +#pragma once +#include "fsa.hh" +#include "syntax.hh" + +enum class ExprTag { + start = 1, + inner = 2, + final = 4, +}; + +extern inline bool has_start(ExprTag x) { return long(x) & long(ExprTag::start); } +extern inline bool has_inner(ExprTag x) { return long(x) & long(ExprTag::inner); } +extern inline bool has_final(ExprTag x) { return long(x) & long(ExprTag::final); } + +bool operator<(ExprTag x, ExprTag y); +bool assoc_has_expr(vector>& as, const Expr* x); + +struct FsaAnno { + bool deterministic; + Fsa fsa; + vector>> assoc; + void accessible(const vector* starts, vector& mapping); + void add_assoc(Expr& expr); + void complement(ComplementExpr* expr); + void co_accessible(const vector* final, vector& mapping); + void concat(FsaAnno& rhs, ConcatExpr* expr); + void determinize(const vector* starts, vector>* mapping); + void difference(FsaAnno& rhs, DifferenceExpr* expr); + void intersect(FsaAnno& rhs, IntersectExpr* expr); + void minimize(vector>* mapping); + void plus(PlusExpr* expr); + void question(QuestionExpr* expr); + void repeat(RepeatExpr& expr); + void star(StarExpr* expr); + void substring_grammar(); + void union_(FsaAnno& rhs, UnionExpr* expr); + static FsaAnno bracket(BracketExpr& expr); + static FsaAnno call(CallExpr& expr); + static FsaAnno collapse(CollapseExpr& expr); + static FsaAnno dot(DotExpr* expr); + static FsaAnno embed(EmbedExpr& expr); + static FsaAnno epsilon_fsa(EpsilonExpr* expr); + static FsaAnno literal(LiteralExpr& expr); +}; diff --git a/yanshi/src/lexer.l b/yanshi/src/lexer.l new file mode 100644 index 0000000..44464e2 --- /dev/null +++ b/yanshi/src/lexer.l @@ -0,0 +1,326 @@ +%{ +#include "lexer_helper.hh" +#include "option.hh" +#include "parser.hh" +#include "syntax.hh" + +#include +#include +#include +#include +#include +using namespace std; + +#define YY_USER_ACTION \ + do { \ + yylloc->start = yyget_extra(yyscanner); \ + yylloc->end = yylloc->start + yyleng; \ + yyset_extra(yylloc->end, yyscanner); \ + } while (0); + +static string tmp_bracket, tmp_str; +static long tmp_str_pos; +static bool semicolon; + +static long invalid_escape(YYSTYPE* yylval, const char* text) +{ + yylval->errmsg = aprintf("invalid \\-escape: %s", text); + return INVALID_CHARACTER; +} + +static int invalid_escape_octonary(YYSTYPE* yylval, const char* text) +{ + yylval->errmsg = aprintf("invalid number after \\-escape: %s", text); + return INVALID_CHARACTER; +} + +static int invalid_escape_x(YYSTYPE* yylval, const char* text) +{ + yylval->errmsg = aprintf("invalid number after \\x-escape: %s", text); + return INVALID_CHARACTER; +} + +static int invalid_escape_u(YYSTYPE* yylval, const char* text) +{ + yylval->errmsg = aprintf("invalid number after \\u-escape: %s", text); + return INVALID_CHARACTER; +} + +static int invalid_escape_U(YYSTYPE* yylval, const char* text) +{ + yylval->errmsg = aprintf("invalid number after \\U-escape: %s", text); + return INVALID_CHARACTER; +} + +static void unexpected_eof(YYSTYPE* yylval, const char* token_end) +{ + yylval->errmsg = aprintf("missing %s at end of file", token_end); +} + +static void unexpected_newline(YYSTYPE* yylval, const char* token_end) +{ + yylval->errmsg = aprintf("missing %s at end of line", token_end); +} + +static int unexpected_codepoint(YYSTYPE* yylval) +{ + yylval->errmsg = aprintf("cannot use Unicode codepoints"); + return INVALID_CHARACTER; +} + +extern "C" int raw_yywrap(yyscan_t yyscanner) +{ + semicolon = false; + return 1; +} +%} + +%option yywrap noinput +%option reentrant +%option extra-type="long" +%option bison-bridge bison-locations +%option prefix="raw_yy" +%option stack + +%x EXPECT_CODE +%x AFTER_ACTION_OP +%x AFTER_EXPORT +%x IN_BRACE +%x IN_CODE +%x IN_COMMENT +%x IN_BRACKET +%x IN_BRACKET_FIRST +%x IN_LINE_COMMENT +%s IN_PAREN +%x IN_Q_STRING +%x IN_QQ_STRING + +D [0-9] +H [0-9A-Fa-f] +L [a-zA-Z_\x80-\xff] + +%% + +"::" return COLONCOLON; +".." return DOTDOT; +"&&" return AMPERAMPER; +";" if (semicolon) return '\n'; +[-~!&*=+,.?|{}:] return yytext[0]; +"action" yy_push_state(EXPECT_CODE, yyscanner); return ACTION; +"as" return AS; +"c++" yy_push_state(EXPECT_CODE, yyscanner); return CPP; +"epsilon" return EPSILON; +"export" yy_push_state(AFTER_EXPORT, yyscanner); return EXPORT; +"import" return IMPORT; +"intact" return INTACT; +"semicolon" semicolon = true; +"nosemicolon" semicolon = false; +{L}({L}|{D})* yylval->str = new string(yytext); return IDENT; +{D}+ yylval->integer = atol(yytext); return INTEGER; +"#define" return PREPROCESS_DEFINE; + +"#" yy_push_state(IN_LINE_COMMENT, yyscanner); +"//" yy_push_state(IN_LINE_COMMENT, yyscanner); +{ + "\n" yy_pop_state(yyscanner); unput('\n'); yyset_extra(yylloc->end-1, yyscanner); + <> yy_pop_state(yyscanner); + . {} +} + +"/*" yy_push_state(IN_COMMENT, yyscanner); +{ + "*/" yy_pop_state(yyscanner); + <> yy_pop_state(yyscanner); + .|\n {} +} + +"(" yy_push_state(IN_PAREN, yyscanner); return '('; +")" { + if (YY_START != IN_PAREN) { + unexpected_newline(yylval, ")"); + return INVALID_CHARACTER; + } + yy_pop_state(yyscanner); + return ')'; +} + +"[" yy_push_state(IN_BRACKET_FIRST, yyscanner); return '['; +{ + "^" BEGIN IN_BRACKET; return '^'; + [^-\\\]\n] { yy_pop_state(yyscanner); yy_push_state(IN_BRACKET, yyscanner); yylval->integer = yytext[0]; return CHAR; } + "-" { yylval->integer = '-'; return CHAR; } +} +{ + "]" { + yy_pop_state(yyscanner); + if (YY_START == INITIAL || YY_START == IN_PAREN) + return ']'; + } + [^-\\\]\n] yylval->integer = yytext[0]; return CHAR; +} +{ + \\[0-7]+ { + BEGIN IN_BRACKET; + long c = strtol(yytext+1, NULL, 8); + if (UCHAR_MAX < c) + return invalid_escape_octonary(yylval, yytext); + yylval->integer = c; + return CHAR; + } + \\u[0-9a-fA-F]+ { + BEGIN IN_BRACKET; + if (opt_bytes) + return unexpected_codepoint(yylval); + long c = strtol(yytext+2, NULL, 16), len = 0; + if (UINT16_MAX < c) + return invalid_escape_u(yylval, yytext); + yylval->integer = c; + return CHAR; + } + \\U[0-9a-fA-F]+ { + BEGIN IN_BRACKET; + if (opt_bytes) + return unexpected_codepoint(yylval); + long c = strtol(yytext+2, NULL, 16), len = 0; + if (MAX_CODEPOINT < c) + return invalid_escape_U(yylval, yytext); + yylval->integer = c; + return CHAR; + } + \\x[0-9a-fA-F]+ { + BEGIN IN_BRACKET; + long c = strtol(yytext+2, NULL, 16); + if (UCHAR_MAX < c) + return invalid_escape_x(yylval, yytext); + yylval->integer = c; + return CHAR; + } + \\a BEGIN IN_BRACKET; yylval->integer = '\a'; return CHAR; + \\b BEGIN IN_BRACKET; yylval->integer = '\b'; return CHAR; + \\f BEGIN IN_BRACKET; yylval->integer = '\f'; return CHAR; + \\n BEGIN IN_BRACKET; yylval->integer = '\n'; return CHAR; + \\r BEGIN IN_BRACKET; yylval->integer = '\r'; return CHAR; + \\t BEGIN IN_BRACKET; yylval->integer = '\t'; return CHAR; + \\v BEGIN IN_BRACKET; yylval->integer = '\v'; return CHAR; + \\. BEGIN IN_BRACKET; yylval->integer = yytext[1]; return CHAR; + - BEGIN IN_BRACKET; return '-'; + "\n" unexpected_newline(yylval, "]"); return INVALID_CHARACTER; + <> yy_pop_state(yyscanner); unexpected_eof(yylval, "]"); +} + +{ // optional 'BRACED_CODE' to specify extra parameters + "intact" yy_pop_state(yyscanner); return INTACT; + {L}({L}|{D})* yy_pop_state(yyscanner); yylval->str = new string(yytext); return IDENT; + "{" BEGIN IN_CODE; tmp_bracket.clear(); + [ \t\n] {} + <> yy_pop_state(yyscanner); +} + +{ + {L}({L}|{D})* yylval->str = new string(yytext); return IDENT; + "{" BEGIN IN_CODE; tmp_bracket.clear(); + [ \t\n] {} + <> yy_pop_state(yyscanner); +} + +[>@%$] yy_push_state(AFTER_ACTION_OP, yyscanner); return yytext[0]; +{ + -?{D}+ yylval->integer = atol(yytext); return INTEGER; + {L}({L}|{D})* yy_pop_state(yyscanner); yylval->str = new string(yytext); return IDENT; + "{" BEGIN IN_CODE; tmp_bracket.clear(); + [ \t\n]+ {} + <> yy_pop_state(yyscanner); + . yylval->errmsg = strdup("invalid character"); return INVALID_CHARACTER; +} +{ + "'" { tmp_bracket += '\''; yy_push_state(IN_Q_STRING, yyscanner); } + "\"" { tmp_bracket += '"'; yy_push_state(IN_QQ_STRING, yyscanner); } + "{" { tmp_bracket += '{'; yy_push_state(IN_CODE, yyscanner); } + "}" { + yy_pop_state(yyscanner); + if (YY_START == INITIAL || YY_START == IN_PAREN) { + yylval->str = new string(tmp_bracket); + return BRACED_CODE; + } else + tmp_bracket += '}'; + } + .|"\n" tmp_bracket += yytext[0]; + <> yy_pop_state(yyscanner); unexpected_eof(yylval, "}"); +} + +' tmp_str.clear(); tmp_str_pos = yylloc->start; yy_push_state(IN_Q_STRING, yyscanner); +"\"" tmp_str.clear(); tmp_str_pos = yylloc->start; yy_push_state(IN_QQ_STRING, yyscanner); +{ + ' { + yy_pop_state(yyscanner); + if (YY_START == INITIAL || YY_START == IN_PAREN) { + yylval->str = new string(tmp_str); + yylloc->start = tmp_str_pos; + return STRING_LITERAL; + } + tmp_bracket += yytext; + } + <> yy_pop_state(yyscanner); unexpected_eof(yylval, "'"); +} +{ + "\"" { + yy_pop_state(yyscanner); + if (YY_START == INITIAL || YY_START == IN_PAREN) { + yylval->str = new string(tmp_str); + yylloc->start = tmp_str_pos; + return STRING_LITERAL; + } + tmp_bracket += yytext; + } + <> yy_pop_state(yyscanner); unexpected_eof(yylval, "\""); +} + +{ + \\[0-7]+ { + long c = strtol(yytext+1, NULL, 8); + if (UCHAR_MAX < c) + return invalid_escape_octonary(yylval, yytext); + tmp_str.push_back(c); + tmp_bracket += yytext; + } + \\x[0-9a-fA-F]+ { + long c = strtol(yytext+2, NULL, 16); + if (UCHAR_MAX < c) + return invalid_escape_x(yylval, yytext); + tmp_str.push_back(c); + tmp_bracket += yytext; + } + \\u[0-9a-fA-F]+ { + char s[4]; + long c = strtol(yytext+2, NULL, 16), len = 0; + if (UINT16_MAX < c) + return invalid_escape_u(yylval, yytext); + U8_APPEND_UNSAFE(s, len, c); + tmp_str.insert(tmp_str.end(), s, s+len); + tmp_bracket += yytext; + } + \\U[0-9a-fA-F]+ { + char s[4]; + long c = strtol(yytext+2, NULL, 16), len = 0; + if (MAX_CODEPOINT < c) + return invalid_escape_U(yylval, yytext); + U8_APPEND_UNSAFE(s, len, c); + tmp_str.insert(tmp_str.end(), s, s+len); + tmp_bracket += yytext; + } + \\a tmp_str += '\a'; tmp_bracket += yytext; + \\b tmp_str += '\b'; tmp_bracket += yytext; + \\f tmp_str += '\f'; tmp_bracket += yytext; + \\n tmp_str += '\n'; tmp_bracket += yytext; + \\r tmp_str += '\r'; tmp_bracket += yytext; + \\t tmp_str += '\t'; tmp_bracket += yytext; + \\v tmp_str += '\v'; tmp_bracket += yytext; + \\[\n\"\'?\\] tmp_str += yytext[1]; tmp_bracket += yytext; + \\. return invalid_escape(yylval, yytext); + .|\n tmp_str += yytext[0]; tmp_bracket += yytext[0]; +} + +\\\n {} +"\n" if (YY_START == INITIAL && ! semicolon) return '\n'; +[ \t]+ {} +. { yylval->errmsg = strdup("invalid character"); return INVALID_CHARACTER; } diff --git a/yanshi/src/lexer_helper.cc b/yanshi/src/lexer_helper.cc new file mode 100644 index 0000000..0627f3f --- /dev/null +++ b/yanshi/src/lexer_helper.cc @@ -0,0 +1,15 @@ +#include "common.hh" + +#include +#include +using namespace std; + +char* aprintf(const char* fmt, ...) +{ + va_list va; + va_start(va, fmt); + char* r = NULL; + vasprintf(&r, fmt, va); + va_end(va); + return r; +} diff --git a/yanshi/src/lexer_helper.hh b/yanshi/src/lexer_helper.hh new file mode 100644 index 0000000..2e6b9b7 --- /dev/null +++ b/yanshi/src/lexer_helper.hh @@ -0,0 +1,4 @@ +#pragma once + +char* aprintf(const char* fmt, ...) + __attribute__((format(printf, 1, 2))); diff --git a/yanshi/src/loader.cc b/yanshi/src/loader.cc new file mode 100644 index 0000000..602c660 --- /dev/null +++ b/yanshi/src/loader.cc @@ -0,0 +1,556 @@ +#include "common.hh" +#include "compiler.hh" +#include "loader.hh" +#include "option.hh" +#include "parser.hh" +#include "repl.hh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +using namespace std; + +static map, Module> inode2module; +static unordered_map> depended_by; // key ranges over all DefineStmt +map> used_as_call, used_as_collapse, used_as_embed; +static DefineStmt* main_export; +Module* main_module; +FILE *output, *output_header; + +void print_module_info(Module& mo) +{ + yellow(); printf("filename: %s\n", mo.filename.c_str()); + cyan(); puts("qualified imports:"); sgr0(); + for (auto& x: mo.qualified_import) + printf(" %s as %s\n", x.second->filename.c_str(), x.first.c_str()); + cyan(); puts("unqualified imports:"); sgr0(); + for (auto& x: mo.unqualified_import) + printf(" %s\n", x->filename.c_str()); + cyan(); puts("defined actions:"); sgr0(); + for (auto& x: mo.defined_action) + printf(" %s\n", x.first.c_str()); + cyan(); puts("defined:"); sgr0(); + for (auto& x: mo.defined) + printf(" %s\n", x.first.c_str()); +} + +Stmt* resolve(Module& mo, const string qualified, const string& ident) +{ + if (qualified.size()) { + if (! mo.qualified_import.count(qualified)) + return NULL; + auto it = mo.qualified_import[qualified]->defined.find(ident); + if (it == mo.qualified_import[qualified]->defined.end()) + return NULL; + return it->second; + } else { + Stmt* r = NULL; + if (mo.macro.count(ident)) + r = mo.macro[ident]; + if (mo.defined.count(ident)) { + if (r) return (Stmt*)1; + r = mo.defined[ident]; + } + for (auto* import: mo.unqualified_import) { + if (import->macro.count(ident)) { + if (r) return (Stmt*)1; + r = import->macro[ident]; + } + if (import->defined.count(ident)) { + if (r) return (Stmt*)1; + r = import->defined[ident]; + } + } + return r; + } +} + +ActionStmt* resolve_action(Module& mo, const string qualified, const string& ident) +{ + if (qualified.size()) { + if (! mo.qualified_import.count(qualified)) + return NULL; + auto it = mo.qualified_import[qualified]->defined_action.find(ident); + if (it == mo.qualified_import[qualified]->defined_action.end()) + return NULL; + return it->second; + } else { + ActionStmt* r = NULL; + if (mo.defined_action.count(ident)) + r = mo.defined_action[ident]; + for (auto* import: mo.unqualified_import) + if (import->defined_action.count(ident)) { + if (r) return (ActionStmt*)1; + r = import->defined_action[ident]; + } + return r; + } +} + +struct ModuleImportDef : PreorderStmtVisitor { + Module& mo; + long& n_errors; + ModuleImportDef(Module& mo, long& n_errors) : mo(mo), n_errors(n_errors) {} + + void visit(ActionStmt& stmt) override { + if (mo.defined_action.count(stmt.ident)) { + n_errors++; + mo.locfile.error(stmt.loc, "redefined '%s'", stmt.ident.c_str()); + } else + mo.defined_action[stmt.ident] = &stmt; + } + // TODO report error: import 'aa.hs' (#define d 3) ; #define d 4 + void visit(DefineStmt& stmt) override { + if (mo.defined.count(stmt.lhs) || mo.macro.count(stmt.lhs)) { + n_errors++; + mo.locfile.error(stmt.loc, "redefined '%s'", stmt.lhs.c_str()); + } else { + mo.defined.emplace(stmt.lhs, &stmt); + stmt.module = &mo; + depended_by[&stmt]; // empty + } + } + void visit(ImportStmt& stmt) override { + Module* m = load_module(n_errors, stmt.filename); + if (! m) { + n_errors++; + mo.locfile.error(stmt.loc, "'%s': %s", stmt.filename.c_str(), errno ? strerror(errno) : "parse error"); + return; + } + if (stmt.qualified.size()) + mo.qualified_import[stmt.qualified] = m; + else if (count(ALL(mo.unqualified_import), m) == 0) + mo.unqualified_import.push_back(m); + } + void visit(PreprocessDefineStmt& stmt) override { + if (mo.defined.count(stmt.ident) || mo.macro.count(stmt.ident)) { + n_errors++; + mo.locfile.error(stmt.loc, "redefined '%s'", stmt.ident.c_str()); + } else + mo.macro[stmt.ident] = &stmt; + } +}; + +struct ModuleUse : PrePostActionExprStmtVisitor { + Module& mo; + long& n_errors; + DefineStmt* stmt = NULL; + ModuleUse(Module& mo, long& n_errors) : mo(mo), n_errors(n_errors) {} + + void pre_expr(Expr& expr) override { + expr.stmt = stmt; + } + + void post_expr(Expr& expr) override { + for (auto a: expr.entering) + PrePostActionExprStmtVisitor::visit(*a.first); + for (auto a: expr.finishing) + PrePostActionExprStmtVisitor::visit(*a.first); + for (auto a: expr.leaving) + PrePostActionExprStmtVisitor::visit(*a.first); + for (auto a: expr.transiting) + PrePostActionExprStmtVisitor::visit(*a.first); + } + + void visit(RefAction& action) override { + ActionStmt* r = resolve_action(mo, action.qualified, action.ident); + if (! r) { + n_errors++; + if (action.qualified.size()) + mo.locfile.error(action.loc, "'%s::%s' undefined", action.qualified.c_str(), action.ident.c_str()); + else + mo.locfile.error(action.loc, "'%s' undefined", action.ident.c_str()); + } else if (r == (Stmt*)1) { + n_errors++; + mo.locfile.error(action.loc, "'%s' redefined", action.ident.c_str()); + } else + action.define_stmt = r; + } + + void visit(BracketExpr& expr) override { + for (auto& x: expr.intervals.to) + AB = max(AB, x.second); + } + void visit(CallExpr& expr) override { + Stmt* r = resolve(mo, expr.qualified, expr.ident); + if (! r) + error_undefined(expr.loc, expr.qualified, expr.ident); + else if (r == (Stmt*)1) + error_ambiguous(expr.loc, expr.ident); + else if (auto d = dynamic_cast(r)) + error_misuse_macro("CallExpr", expr.loc, expr.qualified, expr.ident); + else if (auto d = dynamic_cast(r)) { + used_as_call[d].push_back(&expr); + expr.define_stmt = d; + } else + assert(0); + } + void visit(CollapseExpr& expr) override { + Stmt* r = resolve(mo, expr.qualified, expr.ident); + if (! r) + error_undefined(expr.loc, expr.qualified, expr.ident); + else if (r == (Stmt*)1) + error_ambiguous(expr.loc, expr.ident); + else if (auto d = dynamic_cast(r)) + error_misuse_macro("CollapseExpr", expr.loc, expr.qualified, expr.ident); + else if (auto d = dynamic_cast(r)) { + used_as_collapse[d].push_back(&expr); + expr.define_stmt = d; + } else + assert(0); + } + void visit(DefineStmt& stmt) override { + this->stmt = &stmt; + PrePostActionExprStmtVisitor::visit(*stmt.rhs); + this->stmt = NULL; + } + void visit(EmbedExpr& expr) override { + // introduce dependency + Stmt* r = resolve(mo, expr.qualified, expr.ident); + if (! r) + error_undefined(expr.loc, expr.qualified, expr.ident); + else if (r == (Stmt*)1) + error_ambiguous(expr.loc, expr.ident); + else if (auto d = dynamic_cast(r)) { + // enlarge alphabet + expr.define_stmt = NULL; + expr.macro_value = d->value; + AB = max(AB, d->value+1); + } else if (auto d = dynamic_cast(r)) { + depended_by[d].push_back(stmt); + used_as_embed[d].push_back(&expr); + expr.define_stmt = d; + } else + assert(0); + } +private: + void error_undefined(const Location& loc, const string& qualified, const string& ident) { + n_errors++; + if (qualified.size()) + mo.locfile.error(loc, "'%s::%s' undefined", qualified.c_str(), ident.c_str()); + else + mo.locfile.error(loc, "'%s' undefined", ident.c_str()); + } + void error_ambiguous(const Location& loc, const string& ident) { + n_errors++; + mo.locfile.error(loc, "ambiguous '%s'", ident.c_str()); + } + void error_misuse_macro(const char* name, const Location& loc, const string& qualified, const string& ident) { + n_errors++; + if (qualified.size()) + mo.locfile.error(loc, "macro '%s::%s' used as %s", qualified.c_str(), ident.c_str(), name); + else + mo.locfile.error(loc, "macro '%s' used as %s", ident.c_str(), name); + } +}; + +Module* load_module(long& n_errors, const string& filename) +{ + FILE* file = stdin; + if (filename != "-") { + file = fopen(filename.c_str(), "r"); + for (string& include: opt_include_paths) { + if (file) break; + file = fopen((include+'/'+filename).c_str(), "r"); + } + } + if (! file) { + n_errors++; + return NULL; + } + + pair inode{0, 0}; // stdin -> {0, 0} + if (file != stdin) { + struct stat sb; + if (fstat(fileno(file), &sb) < 0) + err_exit(EX_OSFILE, "fstat '%s'", filename.c_str()); + inode = {sb.st_dev, sb.st_ino}; + } + if (inode2module.count(inode)) { + fclose(file); + return &inode2module[inode]; + } + Module& mo = inode2module[inode]; + + string module{file != stdin ? filename : "main"}; + string::size_type t = module.find('.'); + if (t != string::npos) + module.erase(t, module.size()-t); + + long r; + char buf[BUF_SIZE]; + string data; + while ((r = fread(buf, 1, sizeof buf, file)) > 0) { + data += string(buf, buf+r); + if (r < sizeof buf) break; + } + fclose(file); + if (data.empty() || data.back() != '\n') + data.push_back('\n'); + LocationFile locfile(filename, data); + + Stmt* toplevel = NULL; + mo.locfile = locfile; + mo.filename = filename; + long errors = parse(locfile, toplevel); + if (! toplevel) { + n_errors += errors; + mo.status = BAD; + mo.toplevel = NULL; + return &mo; + } + mo.toplevel = toplevel; + return &mo; +} + +static vector topo_define_stmts(long& n_errors) +{ + vector topo; + vector st; + unordered_map vis; // 0: unvisited; 1: in stack; 2: visited; 3: in a cycle + unordered_map cnt; + function dfs = [&](DefineStmt* u) { + if (vis[u] == 2) + return false; + if (vis[u] == 3) + return true; + if (vis[u] == 1) { + u->module->locfile.error_context(u->loc, "'%s': circular embedding", u->lhs.c_str()); + long i = st.size(); + while (st[i-1] != u) + i--; + st.push_back(st[i-1]); + for (; i < st.size(); i++) { + vis[st[i]] = 3; + fputs(" ", stderr); + st[i]->module->locfile.error_context(st[i]->loc, "required by %s", st[i]->lhs.c_str()); + } + fputs("\n", stderr); + return true; + } + cnt[u] = u->export_ ? 1 : 0; + vis[u] = 1; + st.push_back(u); + bool cycle = false; + for (auto v: depended_by[u]) + if (dfs(v)) + cycle = true; + else + cnt[u] += cnt[v]; + st.pop_back(); + vis[u] = 2; + topo.push_back(u); + return cycle; + }; + for (auto& d: depended_by) + if (! vis[d.first] && dfs(d.first)) // detected cycle + n_errors++; + reverse(ALL(topo)); + if (opt_dump_embed) { + magenta(); printf("=== Embed\n"); sgr0(); + for (auto stmt: topo) + if (cnt[stmt] > 0) + printf("count(%s::%s) = %ld\n", stmt->module->filename.c_str(), stmt->lhs.c_str(), cnt[stmt]); + } + return topo; +} + +long load(const string& filename) +{ + long n_errors = 0; + Module* mo = load_module(n_errors, filename); + if (! mo) { + err_exit(EX_OSFILE, "fopen", filename.c_str()); + return n_errors; + } + if (mo->status == BAD) + return n_errors; + main_module = mo; + + DP(1, "Processing import & def"); + for(;;) { + bool done = true; + for (auto& it: inode2module) + if (it.second.status == UNPROCESSED) { + done = false; + Module& mo = it.second; + mo.status = GOOD; + long old = n_errors; + ModuleImportDef p{mo, n_errors}; + for (Stmt* x = mo.toplevel; x; x = x->next) + x->accept(p); + mo.status = old == n_errors ? GOOD : BAD; + } + if (done) break; + } + if (n_errors) + return n_errors; + + DP(1, "Processing use"); + for (auto& it: inode2module) + if (it.second.status == GOOD) { + Module& mo = it.second; + ModuleUse p{mo, n_errors}; + for (Stmt* x = mo.toplevel; x; x = x->next) + x->accept(p); + } + if (n_errors) + return n_errors; + + // warning: not used solely as CallExpr, CollapseExpr or EmbedExpr + { + auto it0 = used_as_call.begin(), it0e = used_as_call.end(), + it1 = used_as_collapse.begin(), it1e = used_as_collapse.end(), + it2 = used_as_embed.begin(), it2e = used_as_embed.end(); + while (it0 != it0e || it1 != it1e || it2 != it2e) { + long k = 0; + long c = 0; + DefineStmt* x = NULL; + if (it0 != it0e && (! x || it0->first < x)) x = it0->first; + if (it1 != it1e && (! x || it1->first < x)) x = it1->first; + if (it2 != it2e && (! x || it2->first < x)) x = it2->first; + if (it0 != it0e && it0->first == x) c++; + if (it1 != it1e && it1->first == x) c++; + if (it2 != it2e && it2->first == x) c++; + if (c > 1) { + x->module->locfile.warning(x->loc, "'%s' is not used solely as CallExpr, CollapseExpr or EmbedExpr", x->lhs.c_str()); + if (it0 != it0e && it0->first == x) + for (auto* y: it0->second) { + fputs(" ", stderr); + y->stmt->module->locfile.warning_context(y->loc, "required by %s", y->stmt->lhs.c_str()); + } + if (it1 != it1e && it1->first == x) + for (auto* y: it1->second) { + fputs(" ", stderr); + y->stmt->module->locfile.warning_context(y->loc, "required by %s", y->stmt->lhs.c_str()); + } + if (it2 != it2e && it2->first == x) + for (auto* y: it2->second) { + fputs(" ", stderr); + y->stmt->module->locfile.warning_context(y->loc, "required by %s", y->stmt->lhs.c_str()); + } + } + if (it0 != it0e && it0->first == x) ++it0, c++; + if (it1 != it1e && it1->first == x) ++it1, c++; + if (it2 != it2e && it2->first == x) ++it2, c++; + } + } + + if (opt_dump_module) { + magenta(); printf("=== Module\n"); sgr0(); + for (auto& it: inode2module) + if (it.second.status == GOOD) { + Module& mo = it.second; + print_module_info(mo); + } + puts(""); + } + + if (opt_dump_tree) { + magenta(); printf("=== Tree\n"); sgr0(); + StmtPrinter p; + for (auto& it: inode2module) + if (it.second.status == GOOD) { + Module& mo = it.second; + yellow(); printf("filename: %s\n", mo.filename.c_str()); sgr0(); + for (Stmt* x = mo.toplevel; x; x = x->next) + x->accept(p); + } + puts(""); + } + + DP(1, "Topological sorting"); + vector topo = topo_define_stmts(n_errors); + if (n_errors) + return n_errors; + + if (opt_check) + return 0; + + // AB has been updated by ModuleUse + action_label_base = action_label = AB; + call_label_base = call_label = action_label+1000000; + collapse_label_base = collapse_label = call_label+1000000; + + DP(1, "Compiling DefineStmt"); + for (auto stmt: topo) + compile(stmt); + + output = strcmp(opt_output_filename, "-") ? fopen(opt_output_filename, "w") : stdout; + if (! output) { + n_errors++; + err_exit(EX_OSFILE, "fopen", opt_output_filename); + return n_errors; + } + + unordered_map>> stmt2call_addr; + DP(1, "Compiling exporting DefineStmt (coalescing referenced CallExpr/CollapseExpr)"); + for (Stmt* x = main_module->toplevel; x; x = x->next) + if (auto xx = dynamic_cast(x)) + if (xx->export_ && ! compile_export(xx)) + n_errors++; + if (n_errors) + return n_errors; + + for (Stmt* x = main_module->toplevel; x; x = x->next) + if (auto xx = dynamic_cast(x)) + if (xx->export_) { + FsaAnno& anno = compiled[xx]; + if (opt_dump_automaton) + print_automaton(anno.fsa); + if (opt_dump_assoc) + print_assoc(anno); + } + + if (opt_mode == Mode::cxx) { + if (opt_output_header_filename) { + output_header = fopen(opt_output_header_filename, "w"); + if (! output_header) { + n_errors++; + err_exit(EX_OSFILE, "fopen", opt_output_header_filename); + return n_errors; + } + } + DP(1, "Generating C++"); + generate_cxx(mo); + if (output_header) + fclose(output_header); + } else if (opt_mode == Mode::graphviz) { + DP(1, "Generating Graphviz dot"); + generate_graphviz(mo); + } else if (opt_mode == Mode::interactive) { + DP(1, "Testing given string"); + DefineStmt* main_export = NULL; + for (Stmt* x = main_module->toplevel; x; x = x->next) + if (auto xx = dynamic_cast(x)) + if (xx->export_) { + main_export = xx; + break; + } + if (! main_export) + puts("no exporting DefineStmt"); + else { + printf("Testing %s\n", main_export->lhs.c_str()); + repl(main_export); + } + } + + fclose(output); + return n_errors; +} + +void unload_all() +{ + for (auto& it: inode2module) { + Module& mo = it.second; + stmt_free(mo.toplevel); + } +} diff --git a/yanshi/src/loader.hh b/yanshi/src/loader.hh new file mode 100644 index 0000000..c6c529e --- /dev/null +++ b/yanshi/src/loader.hh @@ -0,0 +1,35 @@ +#pragma once +#include "syntax.hh" + +#include +#include +#include +#include +#include +using std::map; +using std::set; +using std::string; +using std::unordered_map; +using std::vector; + +enum ModuleStatus { UNPROCESSED = 0, BAD, GOOD }; + +struct Module { + ModuleStatus status; + LocationFile locfile; + string filename; + Stmt* toplevel; + unordered_map defined; + vector unqualified_import; + unordered_map qualified_import; + unordered_map defined_action; + unordered_map macro; +}; + +Stmt* resolve(Module& mo, const string qualified, const string& ident); +long load(const string& filename); +Module* load_module(long& n_errors, const string& filename); +void unload_all(); +extern Module* main_module; +extern FILE *output, *output_header; +extern map> used_as_call, used_as_collapse, used_as_embed; diff --git a/yanshi/src/location.cc b/yanshi/src/location.cc new file mode 100644 index 0000000..09515b2 --- /dev/null +++ b/yanshi/src/location.cc @@ -0,0 +1,113 @@ +#include "common.hh" +#include "location.hh" + +#include +#include +using namespace std; + +LocationFile::LocationFile(const string& filename, const string& data) : filename(filename), data(data) +{ + // data ends with '\n' + long nlines = count(data.begin(), data.end(), '\n'); + linemap.assign(nlines+1, 0); + long line = 1; + for (long i = 0; i < data.size(); i++) + if (data[i] == '\n') + linemap[line++] = i+1; +} + +void LocationFile::context(const Location& loc) const +{ + long line1, col1, line2, col2; + locate(loc, line1, col1, line2, col2); + if (line1 == line2) { + fputs(" ", stderr); + FOR(i, linemap[line1], line1+1 < linemap.size() ? linemap[line1+1] : data.size()) { + if (i == loc.start) + magenta(); + fputc(data[i], stderr); + if (i+1 == loc.end) + sgr0(); + } + } else { + bool first = true; + fputs(" ", stderr); + FOR(i, linemap[line1], linemap[line1+1]) { + if (i == loc.start) + magenta(); + fputc(data[i], stderr); + } + if (line2-line1 < 8) { + FOR(i, linemap[line1+1], linemap[line2]) { + if (first) { first = false; fputs(" ", stderr); } + fputc(data[i], stderr); + if (data[i] == '\n') first = true; + } + } else { + FOR(i, linemap[line1+1], linemap[line1+4]) { + if (first) { first = false; fputs(" ", stderr); } + fputc(data[i], stderr); + if (data[i] == '\n') first = true; + } + fputs(" ........\n", stderr); + FOR(i, linemap[line2-3], linemap[line2]) { + if (first) { first = false; fputs(" ", stderr); } + fputc(data[i], stderr); + if (data[i] == '\n') first = true; + } + } + FOR(i, linemap[line2], line2+1 < linemap.size() ? linemap[line2+1] : data.size()) { + if (first) { first = false; fputs(" ", stderr); } + fputc(data[i], stderr); + if (i+1 == loc.end) + sgr0(); + } + } +} + +void LocationFile::locate(const Location& loc, long& line1, long& col1, long& line2, long& col2) const +{ + line1 = upper_bound(ALL(linemap), loc.start) - linemap.begin() - 1; + line2 = upper_bound(ALL(linemap), max(loc.end-1, 0L)) - linemap.begin() - 1; + col1 = loc.start - linemap[line1]; + col2 = loc.end - linemap[line2]; +} + +void LocationFile::report_location(const Location& loc) const +{ + long line1, col1, line2, col2; + locate(loc, line1, col1, line2, col2); + yellow(2); + fprintf(stderr, "%s ", filename.c_str()); + cyan(2); + if (line1 == line2) + fprintf(stderr, "%ld:%ld-%ld ", line1+1, col1+1, col2); + else + fprintf(stderr, "%ld-%ld:%ld-%ld ", line1+1, line2+1, col1+1, col2); +} + +void LocationFile::error(const Location& loc, const char* fmt, ...) const +{ + report_location(loc); + red(2); + fprintf(stderr, "error "); + va_list va; + va_start(va, fmt); + vfprintf(stderr, fmt, va); + va_end(va); + fputs("\n", stderr); + sgr0(2); +} + +void LocationFile::warning(const Location& loc, const char* fmt, ...) const +{ + report_location(loc); + yellow(2); + fprintf(stderr, "warning "); + va_list va; + va_start(va, fmt); + vfprintf(stderr, fmt, va); + va_end(va); + fputs("\n", stderr); + sgr0(2); +} diff --git a/yanshi/src/location.hh b/yanshi/src/location.hh new file mode 100644 index 0000000..57d9267 --- /dev/null +++ b/yanshi/src/location.hh @@ -0,0 +1,32 @@ +#pragma once +#include "common.hh" + +#include +#include +#include + +struct Location { long start, end; }; + +struct LocationFile { + std::string filename, data; + std::vector linemap; + + LocationFile() = default; + LocationFile(const std::string& filename, const std::string& data); + LocationFile& operator=(const LocationFile&) = default; + void context(const Location& loc) const; + void locate(const Location& loc, long& line1, long& col1, long& line2, long& col2) const; + void report_location(const Location& loc) const; + void error(const Location& loc, const char* fmt, ...) const; + void warning(const Location& loc, const char* fmt, ...) const; + template + void error_context(const Location& loc, const char* fmt, Args&&... args) const { + error(loc, fmt, std::forward(args)...); + context(loc); + } + template + void warning_context(const Location& loc, const char* fmt, Args&&... args) const { + warning(loc, fmt, std::forward(args)...); + context(loc); + } +}; diff --git a/yanshi/src/main.cc b/yanshi/src/main.cc new file mode 100644 index 0000000..bac7585 --- /dev/null +++ b/yanshi/src/main.cc @@ -0,0 +1,154 @@ +#include "common.hh" +#include "fsa.hh" +#include "loader.hh" +#include "option.hh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +using namespace std; + +void print_help(FILE *fh) +{ + fprintf(fh, "Usage: %s [OPTIONS] dir\n", program_invocation_short_name); + fputs( + "\n" + "Options:\n" + " -b,--bytes make labels range over [0,256), Unicode literals will be treated as UTF-8 bytes\n" + " -C generate C source code (default: C++)\n" + " --check check syntax & use/def\n" + " --debug debug level\n" + " --debug-output filename for debug output\n" + " --dump-action dump associated actions for each edge\n" + " --dump-assoc dump associated AST Expr for each state\n" + " --dump-automaton dump automata\n" + " --dump-embed dump statistics of EmbedExpr\n" + " --dump-module dump module use/def/...\n" + " --dump-tree dump AST\n" + " --extern-c generate extern \"C\" specifier\n" + " -G,--graph output a Graphviz dot file\n" + " -I,--import add to search path for 'import'\n" + " -i,--interactive interactive mode\n" + " --max-return-stack max length of return stack in C generator (default: 100)\n" + " -k,--keep-inaccessible do not perform accessible/co-accessible\n" + " -S,--standalone generate header and 'main()'\n" + " --substring-grammar construct regular approximation of the substring grammar. Inner states of nonterminals labeled 'intact' are not connected to start/final\n" + " -o,--output .cc output filename\n" + " -O,--output-header .hh output filename\n" + " -h, --help display this help and exit\n" + "\n" + , fh); + exit(fh == stdout ? 0 : EX_USAGE); +} + +int main(int argc, char *argv[]) +{ + setlocale(LC_ALL, ""); + int opt; + static struct option long_options[] = { + {"bytes", no_argument, 0, 'b'}, + {"check", required_argument, 0, 'c'}, + {"debug", required_argument, 0, 'd'}, + {"debug-output", required_argument, 0, 'l'}, + {"dump-action", no_argument, 0, 1000}, + {"dump-assoc", no_argument, 0, 1001}, + {"dump-automaton", no_argument, 0, 1002}, + {"dump-embed", no_argument, 0, 1003}, + {"dump-module", no_argument, 0, 1004}, + {"dump-tree", no_argument, 0, 1005}, + {"extern-c", no_argument, 0, 1007}, + {"graph", no_argument, 0, 'G'}, + {"import", required_argument, 0, 'I'}, + {"interactive", no_argument, 0, 'i'}, + {"max-return-stack", required_argument, 0, 1006}, + {"keep-inaccessible", no_argument, 0, 'k'}, + {"standalone", no_argument, 0, 'S'}, + {"substring-grammar", no_argument, 0, 's'}, + {"output", required_argument, 0, 'o'}, + {"output-header", required_argument, 0, 'O'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0}, + }; + + while ((opt = getopt_long(argc, argv, "bCDcd:GhI:ikl:O:o:Ss", long_options, NULL)) != -1) { + switch (opt) { + case 'b': + opt_bytes = true; + AB = 256; + break; + case 'C': + opt_gen_c = true; + break; + case 'D': + break; + case 'c': + opt_check = true; + break; + case 'd': + debug_level = get_long(optarg); + break; + case 'G': + opt_mode = Mode::graphviz; + break; + case 'h': + print_help(stdout); + break; + case 'I': + opt_include_paths.push_back(string(optarg)); + break; + case 'i': + opt_mode = Mode::interactive; + break; + case 'k': + opt_keep_inaccessible = true; + break; + case 'l': + if (debug_file) + err_exit(EX_USAGE, "multiple '-l'"); + debug_file = fopen(optarg, "w"); + if (! debug_file) + err_exit(EX_OSFILE, "fopen"); + break; + case 'O': + opt_output_header_filename = optarg; + break; + case 'o': + opt_output_filename = optarg; + break; + case 'S': + opt_standalone = true; + break; + case 's': + opt_substring_grammar = true; + break; + case 1000: opt_dump_action = true; break; + case 1001: opt_dump_assoc = true; break; + case 1002: opt_dump_automaton = true; break; + case 1003: opt_dump_embed = true; break; + case 1004: opt_dump_module = true; break; + case 1005: opt_dump_tree = true; break; + case 1006: + opt_max_return_stack = get_long(optarg); + break; + case 1007: opt_gen_extern_c = true; break; + case '?': + print_help(stderr); + break; + } + } + if (! debug_file) + debug_file = stderr; + argc -= optind; + argv += optind; + + long n_errors = load(argc ? argv[0] : "-"); + unload_all(); + fclose(debug_file); + return n_errors ? 2 : 0; +} diff --git a/yanshi/src/option.cc b/yanshi/src/option.cc new file mode 100644 index 0000000..96bba32 --- /dev/null +++ b/yanshi/src/option.cc @@ -0,0 +1,13 @@ +#include "common.hh" +#include "option.hh" +#include + +bool opt_bytes, opt_check, opt_dump_action, opt_dump_assoc, opt_dump_automaton, opt_dump_embed, opt_dump_module, opt_dump_tree, opt_gen_c, opt_gen_extern_c, opt_keep_inaccessible, opt_standalone, opt_substring_grammar; + +long AB = MAX_CODEPOINT+1, opt_max_return_stack = 100; +long debug_level = 3; +FILE* debug_file; +const char* opt_output_filename = "-"; +const char* opt_output_header_filename; +Mode opt_mode = Mode::cxx; +vector opt_include_paths; diff --git a/yanshi/src/option.hh b/yanshi/src/option.hh new file mode 100644 index 0000000..b454a12 --- /dev/null +++ b/yanshi/src/option.hh @@ -0,0 +1,13 @@ +#pragma once +#include +#include +using std::string; +using std::vector; + +extern bool opt_bytes, opt_check, opt_dump_action, opt_dump_assoc, opt_dump_automaton, opt_dump_embed, opt_dump_module, opt_dump_tree, opt_gen_c, opt_gen_extern_c, opt_keep_inaccessible, opt_standalone, opt_substring_grammar; +extern long AB, opt_max_return_stack; +extern const char* opt_output_filename; +extern const char* opt_output_header_filename; +enum class Mode {cxx, graphviz, interactive}; +extern Mode opt_mode; +extern vector opt_include_paths; diff --git a/yanshi/src/parser.y b/yanshi/src/parser.y new file mode 100644 index 0000000..b2ea352 --- /dev/null +++ b/yanshi/src/parser.y @@ -0,0 +1,270 @@ +%code requires { +#include "common.hh" +#include "location.hh" +#include "option.hh" +#include "syntax.hh" + +#include +#include + +#define YYINITDEPTH 1000 +#define YYLTYPE Location +#define YYLLOC_DEFAULT(Loc, Rhs, N) \ + do { \ + if (N) { \ + (Loc).start = YYRHSLOC(Rhs, 1).start; \ + (Loc).end = YYRHSLOC(Rhs, N).end; \ + } else { \ + (Loc).start = YYRHSLOC(Rhs, 0).start; \ + (Loc).end = YYRHSLOC(Rhs, 0).end; \ + } \ + } while (0) + +int parse(const LocationFile& locfile, Stmt*& res); +} + +%locations +%error-verbose +%define api.pure + +%parse-param {Stmt*& res} +%parse-param {long& errors} +%parse-param {const LocationFile& locfile} +%parse-param {void** lexer} +%lex-param {Stmt*& res} +%lex-param {long& errors} +%lex-param {const LocationFile& locfile} +%lex-param {void** lexer} + +%union { + long integer; + string* str; + DisjointIntervals* intervals; + Action* action; + Expr* expr; + Stmt* stmt; + char* errmsg; +} +%destructor { delete $$; } +%destructor { delete $$; } +%destructor { delete $$; } +%destructor { delete $$; } +%destructor { delete $$; } + +%token ACTION AMPERAMPER AS COLONCOLON CPP DOTDOT EPSILON EXPORT IMPORT INTACT INVALID_CHARACTER PREPROCESS_DEFINE +%token CHAR INTEGER +%token IDENT +%token BRACED_CODE +%token STRING_LITERAL + +%type action +%type concat_expr difference_expr factor repeat intersect_expr union_expr union_expr2 unop_expr +%type bracket bracket_items +%type define_stmt preprocess stmt stmt_list + +%{ +#include "lexer.hh" + +#define FAIL(loc, errmsg) \ + do { \ + Location l = loc; \ + yyerror(&l, res, errors, locfile, lexer, errmsg); \ + } while (0) + +void yyerror(YYLTYPE* loc, Stmt*& res, long& errors, const LocationFile& locfile, yyscan_t* lexer, const char *errmsg) +{ + errors++; + locfile.error_context(*loc, "%s", errmsg); +} + +int yylex(YYSTYPE* yylval, YYLTYPE* loc, Stmt*& res, long& errors, const LocationFile& locfile, yyscan_t* lexer) +{ + int token = raw_yylex(yylval, loc, *lexer); + if (token == INVALID_CHARACTER) { + FAIL(*loc, yylval->errmsg ? yylval->errmsg : "invalid character"); + free(yylval->errmsg); + } + return token; +} + +#define gen_repeat(x, inner, low, high) \ + if (low < 0) { \ + FAIL(yyloc, "negative"); \ + } \ + if (low > high) { \ + FAIL(yyloc, "low > high"); \ + } \ + x = new RepeatExpr(inner, low, high) +%} + +%% + +toplevel: + stmt_list { res = $1; } + +stmt_list: + %empty { $$ = new EmptyStmt; } + | '\n' stmt_list { $$ = $2; } + | stmt stmt_list { $1->next = $2; $2->prev = $1; $$ = $1; } + | error stmt_list { $$ = $2; } + +stmt: + define_stmt { $$ = $1; } + | preprocess '\n' { $$ = $1; } + | IMPORT STRING_LITERAL AS IDENT '\n' { $$ = new ImportStmt(*$2, *$4); delete $2; delete $4; $$->loc = yyloc; } + | IMPORT STRING_LITERAL '\n' { string t; $$ = new ImportStmt(*$2, t); delete $2; $$->loc = yyloc; } + | ACTION IDENT BRACED_CODE '\n' { $$ = new ActionStmt(*$2, *$3); delete $2; delete $3; $$->loc = yyloc; } + | CPP BRACED_CODE '\n' { $$ = new CppStmt(*$2); delete $2; $$->loc = yyloc; } + +preprocess: + PREPROCESS_DEFINE IDENT INTEGER { $$ = new PreprocessDefineStmt(*$2, $3); delete $2; $$->loc = yyloc; } + +eq: + '=' + | ':' + +define_stmt: + IDENT eq union_expr '\n' { $$ = new DefineStmt(*$1, $3); delete $1; $$->loc = yyloc; } + | IDENT eq '|' union_expr '\n' { $$ = new DefineStmt(*$1, $4); delete $1; $$->loc = yyloc; } + | IDENT eq '\n' union_expr2 '\n' { $$ = new DefineStmt(*$1, $4); delete $1; $$->loc = yyloc; } + | IDENT eq '\n' '|' union_expr2 '\n' { $$ = new DefineStmt(*$1, $5); delete $1; $$->loc = yyloc; } + | EXPORT define_stmt { $$ = $2; ((DefineStmt*)$$)->export_ = true; $$->loc = yyloc; } + | EXPORT BRACED_CODE define_stmt { $$ = $3; ((DefineStmt*)$$)->export_ = true; ((DefineStmt*)$$)->export_params = *$2; delete $2; $$->loc = yyloc; } + | INTACT define_stmt { $$ = $2; ((DefineStmt*)$$)->intact = true; $$->loc = yyloc; } + +union_expr: + intersect_expr { $$ = $1; } + | union_expr '|' intersect_expr { $$ = new UnionExpr($1, $3); $$->loc = yyloc; } + +union_expr2: + intersect_expr { $$ = $1; } + | union_expr2 '|' intersect_expr { $$ = new UnionExpr($1, $3); $$->loc = yyloc; } + | union_expr2 '\n' '|' intersect_expr { $$ = new UnionExpr($1, $4); $$->loc = yyloc; } + +intersect_expr: + difference_expr { $$ = $1; } + | intersect_expr AMPERAMPER difference_expr { $$ = new IntersectExpr($1, $3); $$->loc = yyloc; } + +difference_expr: + concat_expr { $$ = $1; } + | difference_expr '-' concat_expr { $$ = new DifferenceExpr($1, $3); $$->loc = yyloc; } + +concat_expr: + unop_expr { $$ = $1; } + | concat_expr unop_expr { $$ = new ConcatExpr($1, $2); $$->loc = yyloc; } + +unop_expr: + factor { $$ = $1; } + | '~' unop_expr { $$ = new ComplementExpr($2); $$->loc = yyloc; } + +factor: + EPSILON { $$ = new EpsilonExpr; $$->loc = yyloc; } + | IDENT { string t; $$ = new EmbedExpr(t, *$1); delete $1; $$->loc = yyloc; } + | IDENT COLONCOLON IDENT { $$ = new EmbedExpr(*$1, *$3); delete $1; delete $3; $$->loc = yyloc; } + | '!' IDENT { string t; $$ = new CollapseExpr(t, *$2); delete $2; $$->loc = yyloc; } + | '!' IDENT COLONCOLON IDENT { $$ = new CollapseExpr(*$2, *$4); delete $2; delete $4; $$->loc = yyloc; } + | '&' IDENT { string t; $$ = new CallExpr(t, *$2); delete $2; $$->loc = yyloc; } + | '&' IDENT COLONCOLON IDENT { $$ = new CallExpr(*$2, *$4); delete $2; delete $4; $$->loc = yyloc; } + | STRING_LITERAL { $$ = new LiteralExpr(*$1); delete $1; $$->loc = yyloc; } + | '.' { $$ = new DotExpr(); $$->loc = yyloc; } + | INTEGER { + if (opt_bytes && 256 <= $1) { + FAIL(yyloc, "literal integers should be less than 256 in bytes mode"); + $$ = new DotExpr; + } else { + auto t = new DisjointIntervals; + t->emplace($1, $1+1); + $$ = new BracketExpr(t); + $$->loc = yyloc; + } + } + | bracket { $$ = new BracketExpr($1); $$->loc = yyloc; } + | STRING_LITERAL DOTDOT STRING_LITERAL { + i32 c0, c1, i = 0, j = 0; + if (opt_bytes) { + c0 = u8((*$1)[0]); + c1 = u8((*$3)[0]); + i = j = 1; + } else { + U8_NEXT($1->c_str(), i, $1->size(), c0); + U8_NEXT($3->c_str(), j, $3->size(), c1); + } + delete $1; + delete $3; + if (i != $1->size() || j != $3->size()) { + FAIL(yyloc, "endpoints of Unicode range should be of length 1"); + $$ = new DotExpr; + } else if (c0 > c1) { + FAIL(yyloc, "negative Unicode range"); + $$ = new DotExpr; + } else { + auto t = new DisjointIntervals; + t->emplace(c0, c1+1); + $$ = new BracketExpr(t); + $$->loc = yyloc; + } + } + | '(' union_expr ')' { $$ = $2; } + | '(' error ')' { $$ = new DotExpr; } + | repeat { $$ = $1; } + | factor '>' action { $$ = $1; $$->entering.emplace_back($3, 0L); } + | factor '>' INTEGER action { $$ = $1; $$->entering.emplace_back($4, $3); } + | factor '@' action { $$ = $1; $$->finishing.emplace_back($3, 0L); } + | factor '@' INTEGER action { $$ = $1; $$->finishing.emplace_back($4, $3); } + | factor '%' action { $$ = $1; $$->leaving.emplace_back($3, 0L); } + | factor '%' INTEGER action { $$ = $1; $$->leaving.emplace_back($4, $3); } + | factor '$' action { $$ = $1; $$->transiting.emplace_back($3, 0L); } + | factor '$' INTEGER action { $$ = $1; $$->transiting.emplace_back($4, $3); } + | factor '+' { $$ = new PlusExpr($1); $$->loc = yyloc; } + | factor '?' { $$ = new QuestionExpr($1); $$->loc = yyloc; } + | factor '*' { $$ = new StarExpr($1); $$->loc = yyloc; } + +repeat: + factor '{' INTEGER ',' INTEGER '}' { gen_repeat($$, $1, $3, $5); $$->loc = yyloc; } + | factor '{' INTEGER ',' '}' { gen_repeat($$, $1, $3, LONG_MAX); $$->loc = yyloc; } + | factor '{' INTEGER '}' { gen_repeat($$, $1, $3, $3); $$->loc = yyloc; } + | factor '{' ',' INTEGER '}' { gen_repeat($$, $1, 0, $4); $$->loc = yyloc; } + +action: + IDENT { string t; $$ = new RefAction(t, *$1); delete $1; $$->loc = yyloc; } + | IDENT COLONCOLON IDENT { $$ = new RefAction(*$1, *$3); delete $1; delete $3; $$->loc = yyloc; } + | BRACED_CODE { $$ = new InlineAction(*$1); delete $1; $$->loc = yyloc; } + +bracket: + '[' bracket_items ']' { $$ = $2; } + | '[' '^' bracket_items ']' { + $$ = $3; + $$->flip(); + } + +bracket_items: + bracket_items CHAR '-' CHAR { + $$ = $1; + if ($2 > $4) + FAIL(yyloc, "negative range in character class"); + else + $$->emplace($2, $4+1); + } + | bracket_items CHAR { + $$ = $1; + $$->emplace($2, $2+1); + } + | %empty { $$ = new DisjointIntervals; } + +%% + +int parse(const LocationFile& locfile, Stmt*& res) +{ + yyscan_t lexer; + raw_yylex_init_extra(0, &lexer); + YY_BUFFER_STATE buf = raw_yy_scan_bytes(locfile.data.c_str(), locfile.data.size(), lexer); + long errors = 0; + yyparse(res, errors, locfile, &lexer); + raw_yy_delete_buffer(buf, lexer); + raw_yylex_destroy(lexer); + if (errors > 0) { + stmt_free(res); + res = NULL; + } + return errors; +} diff --git a/yanshi/src/repl.cc b/yanshi/src/repl.cc new file mode 100644 index 0000000..2f9bfd5 --- /dev/null +++ b/yanshi/src/repl.cc @@ -0,0 +1,353 @@ +#include "compiler.hh" +#include "fsa_anno.hh" +#include "loader.hh" +#include "parser.hh" +#include "lexer.hh" // after parser.hh +#include "syntax.hh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef HAVE_READLINE +# include +# include +#endif +using namespace std; + +enum class ReplMode {string, integer}; +static ReplMode mode = ReplMode::string; +static const FsaAnno* anno; +static bool quit; + +struct Command +{ + const char* name; + function fn; +} commands[] = { + {".automaton", [](const char*) {print_automaton(anno->fsa); }}, + {".assoc", [](const char*) {print_assoc(*anno); }}, + {".help", + [](const char*) { + fputs("Commands available from the prompt:\n" + " .automaton dump automaton\n" + " .assoc dump associated AST Expr for each state\n" + " .help display this help\n" + " .integer input is a list of non-negative integers, macros(#define) or '' "" quoted strings\n" + " .macro display defined macros\n" + " .string input is a string\n" + " .stmt change target DefineStmt to \n" + " .quit exit interactive mode\n" + , stdout); + }}, + {".integer", + [](const char*) { + mode = ReplMode::integer; puts(".integer mode"); + }}, + {".macro", + [](const char*) { + for (auto& it: main_module->macro) + printf("%s\t%ld\n", it.first.c_str(), it.second->value); + for (auto* import: main_module->unqualified_import) + for (auto& it: import->macro) + printf("%s\t%ld\n", it.first.c_str(), it.second->value); + }}, + {".stmt", + [](const char* arg) { + Stmt* r = resolve(*main_module, "", arg); + if (! r) + printf("'%s' undefined\n", arg); + else if (r == (Stmt*)1) + printf("ambiguous '%s'\n", arg); + else if (auto d = dynamic_cast(r)) + printf("'%s' is a macro\n", arg); + else if (auto d = dynamic_cast(r)) { + anno = &compiled[d]; + printf("%s :: DefineStmt\n", d->lhs.c_str()); + } else + assert(0); + }}, + {".string", + [](const char*) { + mode = ReplMode::string; puts(".string mode"); + } + }, + {".quit", + [](const char*) { + puts("Leaving interactive mode"); + quit = true; + } + }, +}; + +#ifdef HAVE_READLINE +static char* command_completer(const char* text, int state) +{ + static long i = 0; + if (! state) + i = 0; + while (i < LEN(commands)) { + Command* x = &commands[i++]; + if (! strncmp(x->name, text, strlen(text))) + return strdup(x->name); + } + return NULL; +} + +static char* macro_completer(const char* text, int state) +{ + static Stmt* x; + if (! state) + x = main_module->toplevel; + while (x) { + auto xx = dynamic_cast(x); + x = x->next; + if (xx && ! strncmp(xx->ident.c_str(), text, strlen(text))) + return strdup(xx->ident.c_str()); + } + return NULL; +} + +static char* stmt_completer(const char* text, int state) +{ + static Stmt* x; + if (! state) + x = main_module->toplevel; + while (x) { + auto xx = dynamic_cast(x); + x = x->next; + if (xx && ! strncmp(xx->lhs.c_str(), text, strlen(text))) + return strdup(xx->lhs.c_str()); + } + return NULL; +} + +static char** on_complete(const char* text, int start, int end) +{ + rl_attempted_completion_over = 1; + if (! start) + return rl_completion_matches(text, command_completer); + if (6 <= start && ! strncmp(rl_line_buffer, ".stmt ", 6)) + return rl_completion_matches(text, stmt_completer); + if (mode == ReplMode::integer) + return rl_completion_matches(text, macro_completer); + return NULL; +} +#else +char* readline(const char* prompt) +{ + char* r = NULL; + size_t s = 0; + ssize_t n; + fputs(prompt, stdout); + if ((n = getline(&r, &s, stdin)) > 0) + r[n-1] = '\0'; + else { + free(r); + r = NULL; + } + return r; +} +#endif + +static void run_command(char* line) +{ + size_t p = 1; + while (line[p] && isalnum(line[p])) + p++; + Command* com = NULL; + REP(i, LEN(commands)) + if (! strncmp(commands[i].name, line, p)) { + if (com) com = (Command*)1; + else com = &commands[i]; + } + if (! com) + printf("Unknown command '%s'\n", line); + else if (com == (Command*)1) + printf("Ambiguous command '%s'\n", line); + else { + while (line[p] && isspace(line[p])) + p++; + size_t len = strlen(line); + while (len && isspace(line[len-1])) + line[--len] = '\0'; + com->fn(line+p); + } +} + +void repl(DefineStmt* stmt) +{ +#ifdef HAVE_READLINE + rl_attempted_completion_function = on_complete; +#endif + char buf[BUF_SIZE]; + snprintf(buf, sizeof buf, ".stmt %s", stmt->lhs.c_str()); + run_command(buf); + strcpy(buf, ".integer"); + run_command(buf); + strcpy(buf, ".help"); + run_command(buf); + if (! anno) return; + char* line; + stringstream ss; + while ((line = readline("λ ")) != NULL) { +#ifdef HAVE_READLINE + if (line[0]) + add_history(line); +#endif + if (line[0] == '.') { + run_command(line); + free(line); + if (quit) break; + continue; + } + + long u = anno->fsa.start; + + if (mode == ReplMode::string) { + i32 i = 0, len; + long c; + len = strlen(line); + if (anno->fsa.is_final(u)) yellow(1); + else normal_yellow(1); + printf("%ld ", u); sgr0(); + while (i < len) { + U8_NEXT_OR_FFFD(line, i, len, c); + if (iswcntrl(c)) printf("%ld ", c); + else printf("%lc ", wint_t(c)); + u = anno->fsa.transit(u, c); + if (anno->fsa.is_final(u)) yellow(); + else normal_yellow(); + printf("%ld ", u); sgr0(); + if (u < 0) break; + } + } else { + vector input; + int token; + yyscan_t lexer; + raw_yylex_init_extra(0, &lexer); + YY_BUFFER_STATE buf = raw_yy_scan_bytes(line, strlen(line), lexer); + YYSTYPE yylval; + YYLTYPE yylloc; + while (u >= 0 && (token = raw_yylex(&yylval, &yylloc, lexer)) != 0) { + switch (token) { // all tokens with a destructor should be listed + case IDENT: { + Stmt* stmt = resolve(*main_module, "", yylval.str->c_str()); + if (! stmt) { + printf("'%s' undefined", yylval.str->c_str()); + u = -1; + } else if (stmt == (Stmt*)1) { + printf("ambiguous '%s'", yylval.str->c_str()); + u = -1; + } else if (auto d = dynamic_cast(stmt)) + input.push_back(d->value); + else if (auto d = dynamic_cast(stmt)) { + printf("'%s' is not a macro", yylval.str->c_str()); + u = -1; + } else + assert(0); + delete yylval.str; + break; + } + case INTEGER: + input.push_back(yylval.integer); + break; + case STRING_LITERAL: + if (opt_bytes) + for (unsigned char c: *yylval.str) + input.push_back(c); + else + for (i32 c, i = 0; i < yylval.str->size(); ) { + U8_NEXT_OR_FFFD(yylval.str->c_str(), i, yylval.str->size(), c); + input.push_back(c); + } + delete yylval.str; + break; + case BRACED_CODE: + delete yylval.str; + // fall through + default: + printf("invalid token at column %ld-%ld\n", yylloc.start+1, yylloc.end); + u = -1; + break; + } + } + raw_yy_delete_buffer(buf, lexer); + raw_yylex_destroy(lexer); + + if (u >= 0) { + if (anno->fsa.is_final(u)) yellow(1); + else normal_yellow(1); + printf("%ld ", u); sgr0(); + for (long c: input) { + printf("%ld ", c); + u = anno->fsa.transit(u, c); + if (anno->fsa.is_final(u)) yellow(); + else normal_yellow(); + printf("%ld ", u); sgr0(); + if (u < 0) break; + } + } + } + free(line); + puts(""); + if (u >= 0) { + unordered_map> start_finals; + unordered_map>> inners; + for (auto aa: anno->assoc[u]) { + if (has_start(aa.second)) + start_finals[aa.first->stmt].push_back(aa.first->loc.start); + if (has_inner(aa.second)) { + inners[aa.first->stmt].emplace_back(aa.first->loc.start, 1); + inners[aa.first->stmt].emplace_back(aa.first->loc.end, -1); + } + if (has_final(aa.second)) + start_finals[aa.first->stmt].push_back(aa.first->loc.end); + } + vector stmts; + for (auto& it: start_finals) + stmts.push_back(it.first); + for (auto& it: inners) + stmts.push_back(it.first); + // sort DefineStmt by location + sort(ALL(stmts), [](const DefineStmt* x, const DefineStmt* y) { + if (x->module != y->module) + return x->module < y->module; + if (x->loc.start != y->loc.start) + return x->loc.start < y->loc.start; + return x->loc.end < y->loc.end; + }); + stmts.erase(unique(ALL(stmts)), stmts.end()); + for (auto* stmt: stmts) { + auto& start_final = start_finals[stmt]; + auto& inner = inners[stmt]; + sort(ALL(start_final)); + sort(ALL(inner)); + auto it0 = start_final.begin(); + auto it1 = inner.begin(); + long nest = 0; + FOR(i, stmt->loc.start, stmt->loc.end) { + for (; it0 != start_final.end() && *it0 < i; ++it0); + if (it0 != start_final.end() && *it0 == i) { + cyan(); + putchar(':'); + sgr0(); + } + for (; it1 != inner.end() && it1->first <= i; ++it1) + nest += it1->second; + if (nest) + yellow(); + putchar(stmt->module->locfile.data[i]); + if (nest) + sgr0(); + } + } + } + } +} diff --git a/yanshi/src/repl.hh b/yanshi/src/repl.hh new file mode 100644 index 0000000..763e601 --- /dev/null +++ b/yanshi/src/repl.hh @@ -0,0 +1,4 @@ +#pragma once +#include "syntax.hh" + +void repl(DefineStmt* stmt); diff --git a/yanshi/src/syntax.cc b/yanshi/src/syntax.cc new file mode 100644 index 0000000..376c398 --- /dev/null +++ b/yanshi/src/syntax.cc @@ -0,0 +1,10 @@ +#include "syntax.hh" + +void stmt_free(Stmt* stmt) +{ + while (stmt) { + auto x = stmt->next; + delete stmt; + stmt = x; + } +} diff --git a/yanshi/src/syntax.hh b/yanshi/src/syntax.hh new file mode 100644 index 0000000..99bf009 --- /dev/null +++ b/yanshi/src/syntax.hh @@ -0,0 +1,577 @@ +#pragma once +#include "common.hh" +#include "location.hh" + +#include +#include +#include +#include +#include +#include +using std::move; +using std::pair; +using std::string; +using std::vector; + +//// Visitor + +template +struct Visitor; + +template +struct VisitableBase { + virtual void accept(Visitor& visitor) = 0; +}; + +template +struct Visitable : Base { + void accept(Visitor& visitor) override { + visitor.visit(static_cast(*this)); + } +}; + +struct Action; +struct InlineAction; +struct RefAction; +template<> +struct Visitor { + virtual void visit(Action& action) = 0; + virtual void visit(InlineAction&) = 0; + virtual void visit(RefAction&) = 0; +}; + +struct Expr; +struct BracketExpr; +struct CallExpr; +struct CollapseExpr; +struct ComplementExpr; +struct ConcatExpr; +struct DifferenceExpr; +struct DotExpr; +struct EmbedExpr; +struct EpsilonExpr; +struct IntersectExpr; +struct LiteralExpr; +struct PlusExpr; +struct RepeatExpr; +struct QuestionExpr; +struct StarExpr; +struct UnionExpr; +template<> +struct Visitor { + virtual void visit(Expr&) = 0; + virtual void visit(BracketExpr&) = 0; + virtual void visit(CallExpr&) = 0; + virtual void visit(CollapseExpr&) = 0; + virtual void visit(ComplementExpr&) = 0; + virtual void visit(ConcatExpr&) = 0; + virtual void visit(DifferenceExpr&) = 0; + virtual void visit(DotExpr&) = 0; + virtual void visit(EmbedExpr&) = 0; + virtual void visit(EpsilonExpr&) = 0; + virtual void visit(IntersectExpr&) = 0; + virtual void visit(LiteralExpr&) = 0; + virtual void visit(PlusExpr&) = 0; + virtual void visit(RepeatExpr&) = 0; + virtual void visit(QuestionExpr&) = 0; + virtual void visit(StarExpr&) = 0; + virtual void visit(UnionExpr&) = 0; +}; + +struct Stmt; +struct ActionStmt; +struct CppStmt; +struct DefineStmt; +struct EmptyStmt; +struct ImportStmt; +struct PreprocessDefineStmt; +template<> +struct Visitor { + virtual void visit(Stmt&) = 0; + virtual void visit(ActionStmt&) = 0; + virtual void visit(CppStmt&) = 0; + virtual void visit(DefineStmt&) = 0; + virtual void visit(EmptyStmt&) = 0; + virtual void visit(ImportStmt&) = 0; + virtual void visit(PreprocessDefineStmt&) = 0; +}; + +//// Action + +struct Action : VisitableBase { + Location loc; + virtual ~Action() = default; +}; + +struct InlineAction : Visitable { + string code; + InlineAction(string& code) : code(move(code)) {} +}; + +struct Module; +struct RefAction : Visitable { + string qualified, ident; + ActionStmt* define_stmt; // set by ModuleUse + RefAction(string& qualified, string& ident) : qualified(move(qualified)), ident(move(ident)) {} +}; + +//// Expr + +struct Expr : VisitableBase { + Location loc; + long pre, post, depth; // set by Compiler + vector anc; // set by Compiler + vector> entering, finishing, leaving, transiting; + DefineStmt* stmt = NULL; // set by ModuleImportDef + virtual ~Expr() { + for (auto a: entering) + delete a.first; + for (auto a: finishing) + delete a.first; + for (auto a: leaving) + delete a.first; + for (auto a: transiting) + delete a.first; + } + string name() const { + int status; + std::unique_ptr r{ + abi::__cxa_demangle(typeid(*this).name(), NULL, NULL, &status), + free + }; + std::string t = r.get(); + t = t.substr(0, t.size()-4); // suffix 'Expr' + return t; + } + bool no_action() const { + return entering.empty() && finishing.empty() && leaving.empty() && transiting.empty(); + } +}; + +struct BracketExpr : Visitable { + DisjointIntervals intervals; + BracketExpr(DisjointIntervals* intervals) : intervals(std::move(*intervals)) { delete intervals; } +}; + +struct CallExpr : Visitable { + string qualified, ident; + DefineStmt* define_stmt = NULL; // set by ModuleUse + CallExpr(string& qualified, string& ident) : qualified(move(qualified)), ident(move(ident)) {} +}; + +struct CollapseExpr : Visitable { + string qualified, ident; + DefineStmt* define_stmt = NULL; // set by ModuleUse + CollapseExpr(string& qualified, string& ident) : qualified(move(qualified)), ident(move(ident)) {} +}; + +struct ComplementExpr : Visitable { + Expr* inner; + ComplementExpr(Expr* inner) : inner(inner) {} + ~ComplementExpr() { + delete inner; + } +}; + +struct ConcatExpr : Visitable { + Expr *lhs, *rhs; + ConcatExpr(Expr* lhs, Expr* rhs) : lhs(lhs), rhs(rhs) {} + ~ConcatExpr() { + delete lhs; + delete rhs; + } +}; + +struct DifferenceExpr : Visitable { + Expr *lhs, *rhs; + DifferenceExpr(Expr* lhs, Expr* rhs) : lhs(lhs), rhs(rhs) {} + ~DifferenceExpr() { + delete lhs; + delete rhs; + } +}; + +struct DotExpr : Visitable {}; + +struct EmbedExpr : Visitable { + string qualified, ident; + DefineStmt* define_stmt = NULL; // set by ModuleUse + long macro_value; // set by ModuleUse + EmbedExpr(string& qualified, string& ident) : qualified(move(qualified)), ident(move(ident)) {} +}; + +struct EpsilonExpr : Visitable {}; + +struct IntersectExpr : Visitable { + Expr *lhs, *rhs; + IntersectExpr(Expr* lhs, Expr* rhs) : lhs(lhs), rhs(rhs) {} + ~IntersectExpr() { + delete lhs; + delete rhs; + } +}; + +struct LiteralExpr : Visitable { + string literal; + LiteralExpr(string& literal) : literal(move(literal)) {} +}; + +struct PlusExpr : Visitable { + Expr* inner; + PlusExpr(Expr* inner) : inner(inner) {} + ~PlusExpr() { + delete inner; + } +}; + +struct RepeatExpr : Visitable { + Expr* inner; + long low, high; + RepeatExpr(Expr* inner, long low, long high) : inner(inner), low(low), high(high) {} + ~RepeatExpr() { + delete inner; + } +}; + +struct QuestionExpr : Visitable { + Expr* inner; + QuestionExpr(Expr* inner) : inner(inner) {} + ~QuestionExpr() { + delete inner; + } +}; + +struct StarExpr : Visitable { + Expr* inner; + StarExpr(Expr* inner) : inner(inner) {} + ~StarExpr() { + delete inner; + } +}; + +struct UnionExpr : Visitable { + Expr *lhs, *rhs; + UnionExpr(Expr* lhs, Expr* rhs) : lhs(lhs), rhs(rhs) {} + ~UnionExpr() { + delete lhs; + delete rhs; + } +}; + +//// Stmt + +struct Stmt { + Location loc; + Stmt *prev = NULL, *next = NULL; + virtual ~Stmt() = default; + virtual void accept(Visitor& visitor) = 0; +}; + +struct EmptyStmt : Visitable {}; + +struct ActionStmt : Visitable { + string ident, code; + ActionStmt(string& ident, string& code) : ident(move(ident)), code(move(code)) {} +}; + +struct CppStmt : Visitable { + string code; + CppStmt(string& code) : code(move(code)) {} +}; + +struct DefineStmt : Visitable { + bool export_ = false, intact = false; + string export_params, lhs; + Expr* rhs; + Module* module; // used in topological sort + DefineStmt(string& lhs, Expr* rhs) : lhs(move(lhs)), rhs(rhs) {} + ~DefineStmt() { + delete rhs; + } +}; + +struct ImportStmt : Visitable { + string filename, qualified; + ImportStmt(string& filename, string& qualified) : filename(move(filename)), qualified(move(qualified)) {} +}; + +struct PreprocessDefineStmt : Visitable { + string ident; + long value; + PreprocessDefineStmt(string& ident, long value) : ident(move(ident)), value(value) {} +}; + +void stmt_free(Stmt* stmt); + +//// Visitor implementations + +struct StmtPrinter : Visitor, Visitor, Visitor { + int depth = 0; + + void visit(Action& action) override { + action.accept(*this); + } + void visit(InlineAction& action) override { + printf("%*s%s\n", 2*depth, "", "InlineAction"); + printf("%*s%s\n", 2*(depth+1), "", action.code.c_str()); + } + void visit(RefAction& action) override { + printf("%*s%s\n", 2*depth, "", "RefAction"); + printf("%*s%s\n", 2*(depth+1), "", action.ident.c_str()); + } + + void visit(Expr& expr) override { + if (expr.entering.size()) { + printf("%*s%s\n", 2*depth, "", "@entering"); + depth++; + for (auto a: expr.entering) { + indent(stdout, depth); + printf("%ld\n", a.second); + a.first->accept(*this); + } + depth--; + } + if (expr.finishing.size()) { + printf("%*s%s\n", 2*depth, "", "@finishing"); + depth++; + for (auto a: expr.finishing) { + indent(stdout, depth); + printf("%ld\n", a.second); + a.first->accept(*this); + } + depth--; + } + if (expr.leaving.size()) { + printf("%*s%s\n", 2*depth, "", "@entering"); + depth++; + for (auto a: expr.leaving) { + indent(stdout, depth); + printf("%ld\n", a.second); + a.first->accept(*this); + } + depth--; + } + if (expr.transiting.size()) { + printf("%*s%s\n", 2*depth, "", "@transiting"); + depth++; + for (auto a: expr.transiting) { + indent(stdout, depth); + printf("%ld\n", a.second); + a.first->accept(*this); + } + depth--; + } + expr.accept(*this); + } + void visit(BracketExpr& expr) override { + printf("%*s%s\n", 2*depth, "", "BracketExpr"); + printf("%*s", 2*(depth+1), ""); + for (auto& x: expr.intervals.to) + printf("(%ld,%ld) ", x.first, x.second); + puts(""); + } + void visit(CallExpr& expr) override { + printf("%*s%s\n", 2*depth, "", "CallExpr"); + printf("%*s", 2*(depth+1), ""); + if (expr.qualified.size()) + printf("%s::%s\n", expr.qualified.c_str(), expr.ident.c_str()); + else + printf("%s\n", expr.ident.c_str()); + } + void visit(CollapseExpr& expr) override { + printf("%*s%s\n", 2*depth, "", "CollapseExpr"); + printf("%*s", 2*(depth+1), ""); + if (expr.qualified.size()) + printf("%s::%s\n", expr.qualified.c_str(), expr.ident.c_str()); + else + printf("%s\n", expr.ident.c_str()); + } + void visit(ComplementExpr& expr) override { + printf("%*s%s\n", 2*depth, "", "ComplementExpr"); + depth++; + visit(*expr.inner); + depth--; + } + void visit(ConcatExpr& expr) override { + printf("%*s%s\n", 2*depth, "", "ConcatExpr"); + depth++; + visit(*expr.lhs); + visit(*expr.rhs); + depth--; + } + void visit(DifferenceExpr& expr) override { + printf("%*s%s\n", 2*depth, "", "DifferenceExpr"); + depth++; + visit(*expr.lhs); + visit(*expr.rhs); + depth--; + } + void visit(DotExpr& expr) override { + printf("%*s%s\n", 2*depth, "", "DotExpr"); + } + void visit(EmbedExpr& expr) override { + printf("%*s%s\n", 2*depth, "", "EmbedExpr"); + printf("%*s", 2*(depth+1), ""); + if (expr.qualified.size()) + printf("%s::%s\n", expr.qualified.c_str(), expr.ident.c_str()); + else + printf("%s\n", expr.ident.c_str()); + } + void visit(EpsilonExpr& expr) override { + printf("%*s%s\n", 2*depth, "", "EpsilonExpr"); + } + void visit(IntersectExpr& expr) override { + printf("%*s%s\n", 2*depth, "", "IntersectExpr"); + depth++; + visit(*expr.lhs); + visit(*expr.rhs); + depth--; + } + void visit(LiteralExpr& expr) override { + printf("%*s%s\n", 2*depth, "", "LiteralExpr"); + printf("%*s%s\n", 2*(depth+1), "", expr.literal.c_str()); + } + void visit(PlusExpr& expr) override { + printf("%*s%s\n", 2*depth, "", "PlusExpr"); + depth++; + visit(*expr.inner); + depth--; + } + void visit(RepeatExpr& expr) override { + printf("%*s%s\n", 2*depth, "", "RepeatExpr"); + printf("%*s%ld,%ld\n", 2*(depth+1), "", expr.low, expr.high); + depth++; + visit(*expr.inner); + depth--; + } + void visit(QuestionExpr& expr) override { + printf("%*s%s\n", 2*depth, "", "QuestionExpr"); + depth++; + visit(*expr.inner); + depth--; + } + void visit(StarExpr& expr) override { + printf("%*s%s\n", 2*depth, "", "StarExpr"); + depth++; + visit(*expr.inner); + depth--; + } + void visit(UnionExpr& expr) override { + printf("%*s%s\n", 2*depth, "", "UnionExpr"); + depth++; + visit(*expr.lhs); + visit(*expr.rhs); + depth--; + } + + void visit(Stmt& stmt) override { + stmt.accept(*this); + } + void visit(ActionStmt& stmt) override { + printf("%*s%s\n", 2*depth, "", "ActionStmt"); + printf("%*s%s\n", 2*(depth+1), "", stmt.ident.c_str()); + printf("%*s%s\n", 2*(depth+1), "", stmt.code.c_str()); + } + void visit(CppStmt& stmt) override { + printf("%*s%s\n", 2*depth, "", "CppStmt"); + printf("%*s%s\n", 2*(depth+1), "", stmt.code.c_str()); + } + void visit(DefineStmt& stmt) override { + printf("%*s%s%s\n", 2*depth, "", "DefineStmt", stmt.export_ ? " export" : ""); + depth++; + indent(stdout, depth); + if (stmt.export_params.size()) + printf("(%s) ", stmt.export_params.c_str()); + printf("%s\n", stmt.lhs.c_str()); + visit(*stmt.rhs); + depth--; + } + void visit(EmptyStmt& stmt) override { + printf("%*s%s\n", 2*depth, "", "EmptyStmt"); + } + void visit(ImportStmt& stmt) override { + printf("%*s%s\n", 2*depth, "", "ImportStmt"); + printf("%*s%s\n", 2*(depth+1), "", stmt.filename.c_str()); + if (stmt.qualified.size()) + printf("%*sas %s\n", 2*(depth+1), "", stmt.qualified.c_str()); + } + void visit(PreprocessDefineStmt& stmt) override { + printf("%*s%s\n", 2*depth, "", "PreprocessDefineStmt"); + printf("%*s%s %ld\n", 2*(depth+1), "", stmt.ident.c_str(), stmt.value); + } +}; + +//// Visitor implementation + +struct PreorderStmtVisitor : Visitor { + void visit(Stmt& stmt) override { stmt.accept(*this); } + void visit(ActionStmt& stmt) override {} + void visit(CppStmt& stmt) override {} + void visit(DefineStmt& stmt) override {} + void visit(EmptyStmt& stmt) override {} + void visit(ImportStmt& stmt) override {} + void visit(PreprocessDefineStmt&) override {} +}; + +struct PrePostActionExprStmtVisitor : Visitor, Visitor, Visitor { + virtual void pre_action(Action&) {} + virtual void post_action(Action&) {} + virtual void pre_expr(Expr&) {} + virtual void post_expr(Expr&) {} + virtual void pre_stmt(Stmt&) {} + virtual void post_stmt(Stmt&) {} + + void visit(Action& action) override { + pre_action(action); + action.accept(*this); + post_action(action); + } + void visit(InlineAction&) override {} + void visit(RefAction&) override {} + + void visit(Expr& expr) override { + pre_expr(expr); + expr.accept(*this); + post_expr(expr); + } + void visit(BracketExpr& expr) override {} + void visit(CallExpr& expr) override {} + void visit(CollapseExpr& expr) override {} + void visit(ComplementExpr& expr) override { visit(*expr.inner); } + void visit(ConcatExpr& expr) override { + visit(*expr.lhs); + visit(*expr.rhs); + } + void visit(DifferenceExpr& expr) override { + visit(*expr.lhs); + visit(*expr.rhs); + } + void visit(DotExpr& expr) override {} + void visit(EmbedExpr& expr) override {} + void visit(EpsilonExpr& expr) override {} + void visit(IntersectExpr& expr) override { + visit(*expr.lhs); + visit(*expr.rhs); + } + void visit(LiteralExpr& expr) override {} + void visit(PlusExpr& expr) override { visit(*expr.inner); } + void visit(RepeatExpr& expr) override { visit(*expr.inner); } + void visit(QuestionExpr& expr) override { visit(*expr.inner); } + void visit(StarExpr& expr) override { visit(*expr.inner); } + void visit(UnionExpr& expr) override { + visit(*expr.lhs); + visit(*expr.rhs); + } + + void visit(Stmt& stmt) override { + pre_stmt(stmt); + stmt.accept(*this); + post_stmt(stmt); + } + void visit(ActionStmt& stmt) override {} + void visit(CppStmt& stmt) override {} + void visit(DefineStmt& stmt) override { stmt.rhs->accept(*this); } + void visit(EmptyStmt& stmt) override {} + void visit(ImportStmt& stmt) override {} + void visit(PreprocessDefineStmt&) override {} +}; diff --git a/yanshi/unittest/determinize_test.cc b/yanshi/unittest/determinize_test.cc new file mode 100644 index 0000000..6f20f0b --- /dev/null +++ b/yanshi/unittest/determinize_test.cc @@ -0,0 +1,41 @@ +#include "fsa.hh" +#include "unittest/unittest_helper.hh" + +#include +#include +#include +#include +#include +#include +using namespace std; + +const char test[] = +"4 7 2\n" +"2 3 \n" +"0 0 1\n" +"0 -1 2\n" +"1 1 1\n" +"1 1 3\n" +"2 -1 1\n" +"2 0 3\n" +"3 0 2\n" +; + +int main(int argc, char *argv[]) +{ + if (argc == 1) { + char filename[] = "/tmp/XXXXXX"; + int fd = mkstemp(filename); + write(fd, test, sizeof test-1); + close(fd); + freopen(filename, "r", stdin); + unlink(filename); + } + + auto relate = [](const vector&) {}; + Fsa fsa = read_nfa().determinize(relate); + print_fsa(fsa); + + if (argc == 1) + return fsa.n() == 4 ? 0 : 1; +} diff --git a/yanshi/unittest/difference_test.cc b/yanshi/unittest/difference_test.cc new file mode 100644 index 0000000..bb61747 --- /dev/null +++ b/yanshi/unittest/difference_test.cc @@ -0,0 +1,44 @@ +#include "fsa.hh" +#include "unittest/unittest_helper.hh" + +#include +#include +#include +#include +#include +using namespace std; + +const char test[] = +"4 4 1\n" +"3 \n" +"0 0 1\n" +"0 1 2\n" +"1 0 3\n" +"2 1 3\n" + +"4 4 1\n" +"3 \n" +"0 0 1\n" +"0 1 2\n" +"1 1 3\n" +"2 1 3\n" +; + +int main(int argc, char *argv[]) +{ + if (argc == 1) { + char filename[] = "/tmp/XXXXXX"; + int fd = mkstemp(filename); + write(fd, test, sizeof test-1); + close(fd); + freopen(filename, "r", stdin); + unlink(filename); + } + + auto relate = [](long u, long v) {}; + Fsa a = read_dfa(), b = read_dfa(), fsa = a.difference(b, relate); + print_fsa(fsa); + + if (argc == 1) + return 0; // fsa.n() == 4 ? 0 : 1; +} diff --git a/yanshi/unittest/intersection_test.cc b/yanshi/unittest/intersection_test.cc new file mode 100644 index 0000000..3cf86dd --- /dev/null +++ b/yanshi/unittest/intersection_test.cc @@ -0,0 +1,44 @@ +#include "fsa.hh" +#include "unittest/unittest_helper.hh" + +#include +#include +#include +#include +#include +using namespace std; + +const char test[] = +"4 4 1\n" +"3 \n" +"0 0 1\n" +"0 1 2\n" +"1 0 3\n" +"2 1 3\n" + +"4 4 1\n" +"3 \n" +"0 0 1\n" +"0 1 2\n" +"1 1 3\n" +"2 1 3\n" +; + +int main(int argc, char *argv[]) +{ + if (argc == 1) { + char filename[] = "/tmp/XXXXXX"; + int fd = mkstemp(filename); + write(fd, test, sizeof test-1); + close(fd); + freopen(filename, "r", stdin); + unlink(filename); + } + + auto relate = [](long u, long v) {}; + Fsa a = read_dfa(), b = read_dfa(), fsa = a.intersect(b, relate); + print_fsa(fsa); + + if (argc == 1) + return 0; // fsa.n() == 4 ? 0 : 1; +} diff --git a/yanshi/unittest/minimize_test.cc b/yanshi/unittest/minimize_test.cc new file mode 100644 index 0000000..70e8ce1 --- /dev/null +++ b/yanshi/unittest/minimize_test.cc @@ -0,0 +1,39 @@ +#include "fsa.hh" +#include "unittest/unittest_helper.hh" + +#include +#include +#include +#include +#include +#include +#include +using namespace std; + +const char test[] = +"4 4 1\n" +"3 \n" +"0 0 1\n" +"0 1 2\n" +"1 0 3\n" +"2 0 3\n" +; + +int main(int argc, char *argv[]) +{ + if (argc == 1) { + char filename[] = "/tmp/XXXXXX"; + int fd = mkstemp(filename); + write(fd, test, sizeof test-1); + close(fd); + freopen(filename, "r", stdin); + unlink(filename); + } + + auto relate = [](const vector&) {}; + Fsa fsa = read_dfa().minimize(relate); + print_fsa(fsa); + + if (argc == 1) + return fsa.n() == 3 ? 0 : 1; +} diff --git a/yanshi/unittest/union_test.cc b/yanshi/unittest/union_test.cc new file mode 100644 index 0000000..a300212 --- /dev/null +++ b/yanshi/unittest/union_test.cc @@ -0,0 +1,46 @@ +#include "fsa.hh" +#include "unittest/unittest_helper.hh" + +#include +#include +#include +#include +#include +#include +#include +using namespace std; + +const char test[] = +"4 4 1\n" +"3 \n" +"0 0 1\n" +"0 1 2\n" +"1 0 3\n" +"2 1 3\n" + +"4 4 1\n" +"3 \n" +"0 0 1\n" +"0 1 2\n" +"1 1 3\n" +"2 1 3\n" +; + +int main(int argc, char *argv[]) +{ + if (argc == 1) { + char filename[] = "/tmp/XXXXXX"; + int fd = mkstemp(filename); + write(fd, test, sizeof test-1); + close(fd); + freopen(filename, "r", stdin); + unlink(filename); + } + + auto relate = [](long u, long v) {}; + Fsa a = read_dfa(), b = read_dfa(), fsa = a.union_(b, relate); + print_fsa(fsa); + + if (argc == 1) + return fsa.n() == 4 ? 0 : 1; +} diff --git a/yanshi/unittest/unittest_helper.hh b/yanshi/unittest/unittest_helper.hh new file mode 100644 index 0000000..8ea8bd4 --- /dev/null +++ b/yanshi/unittest/unittest_helper.hh @@ -0,0 +1,68 @@ +#pragma once +#include "common.hh" +#include "fsa.hh" + +#include +#include +#include +#include +#include +using namespace std; + +static Fsa read_nfa() +{ + long n, m, k, u, a, v; + cin >> n >> m >> k; + Fsa r; + r.start = 0; + r.adj.resize(n); + while (k--) { + cin >> u; + r.finals.push_back(u); + } + sort(ALL(r.finals)); + while (m--) { + cin >> u >> a >> v; + if (u < 0 || u >= n) + errx(EX_DATAERR, "%ld: 0 <= u < n", u); + if (a < -1 || a >= 256) + errx(EX_DATAERR, "%ld: -1 <= c < 256", a); + if (v < 0 || v >= n) + errx(EX_DATAERR, "%ld: 0 <= v < n", v); + r.adj[u].emplace_back(a, v); + } + assert(cin.good()); + REP(i, n) + sort(ALL(r.adj[i])); + return r; +} + +static Fsa read_dfa() +{ + Fsa r = read_nfa(); + REP(i, r.n()) + if (r.adj[i].size()) { + if (r.adj[i][0].first < 0) + errx(EX_DATAERR, "epsilon edge found for %ld", i); + REP(j, r.adj[i].size()-1) + if (r.adj[i][j].first == r.adj[i][j+1].first) + errx(EX_DATAERR, "duplicate labels %ld found for %ld", r.adj[i][j].first, i); + } + assert(cin.good()); + return r; +} + +static void print_fsa(const Fsa& fsa) +{ + printf("finals:"); + for (long i: fsa.finals) + printf(" %ld", i); + puts(""); + puts("edges:"); + REP(i, fsa.n()) { + printf("%ld:", i); + for (auto& x: fsa.adj[i]) + printf(" (%ld,%ld)", x.first, x.second); + puts(""); + } +}