diff --git a/.all-contributorsrc b/.all-contributorsrc deleted file mode 100644 index 2f2d00bd..00000000 --- a/.all-contributorsrc +++ /dev/null @@ -1,768 +0,0 @@ -{ - "projectName": "kubekey", - "projectOwner": "kubesphere", - "repoType": "github", - "repoHost": "https://github.com", - "files": [ - "README.md", - "CONTRIBUTORS.md", - "README_zh-CN.md" - ], - "imageSize": 100, - "commit": true, - "commitConvention": "none", - "contributors": [ - { - "login": "pixiake", - "name": "pixiake", - "avatar_url": "https://avatars0.githubusercontent.com/u/22290449?v=4", - "profile": "https://github.com/pixiake", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "Forest-L", - "name": "Forest", - "avatar_url": "https://avatars2.githubusercontent.com/u/50984129?v=4", - "profile": "https://github.com/Forest-L", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "rayzhou2017", - "name": "rayzhou2017", - "avatar_url": "https://avatars2.githubusercontent.com/u/28859385?v=4", - "profile": "https://kubesphere.io/", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "shaowenchen", - "name": "shaowenchen", - "avatar_url": "https://avatars2.githubusercontent.com/u/43693241?v=4", - "profile": "https://www.chenshaowen.com/", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "LinuxSuRen", - "name": "Zhao Xiaojie", - "avatar_url": "https://avatars1.githubusercontent.com/u/1450685?v=4", - "profile": "http://surenpi.com/", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "zackzhangkai", - "name": "Zack Zhang", - "avatar_url": "https://avatars1.githubusercontent.com/u/20178386?v=4", - "profile": "https://github.com/zackzhangkai", - "contributions": [ - "code" - ] - }, - { - "login": "akhilerm", - "name": "Akhil Mohan", - "avatar_url": "https://avatars1.githubusercontent.com/u/7610845?v=4", - "profile": "https://akhilerm.com/", - "contributions": [ - "code" - ] - }, - { - "login": "FeynmanZhou", - "name": "pengfei", - "avatar_url": "https://avatars3.githubusercontent.com/u/40452856?v=4", - "profile": "https://github.com/FeynmanZhou", - "contributions": [ - "doc" - ] - }, - { - "login": "min-zh", - "name": "min zhang", - "avatar_url": "https://avatars1.githubusercontent.com/u/35321102?v=4", - "profile": "https://github.com/min-zh", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "zgldh", - "name": "zgldh", - "avatar_url": "https://avatars1.githubusercontent.com/u/312404?v=4", - "profile": "https://github.com/zgldh", - "contributions": [ - "code" - ] - }, - { - "login": "xrjk", - "name": "xrjk", - "avatar_url": "https://avatars0.githubusercontent.com/u/16330256?v=4", - "profile": "https://github.com/xrjk", - "contributions": [ - "code" - ] - }, - { - "login": "stoneshi-yunify", - "name": "yonghongshi", - "avatar_url": "https://avatars2.githubusercontent.com/u/70880165?v=4", - "profile": "https://github.com/stoneshi-yunify", - "contributions": [ - "code" - ] - }, - { - "login": "shenhonglei", - "name": "Honglei", - "avatar_url": "https://avatars2.githubusercontent.com/u/20896372?v=4", - "profile": "https://github.com/shenhonglei", - "contributions": [ - "doc" - ] - }, - { - "login": "liucy1983", - "name": "liucy1983", - "avatar_url": "https://avatars2.githubusercontent.com/u/2360302?v=4", - "profile": "https://github.com/liucy1983", - "contributions": [ - "code" - ] - }, - { - "login": "lilien1010", - "name": "Lien", - "avatar_url": "https://avatars1.githubusercontent.com/u/3814966?v=4", - "profile": "https://github.com/lilien1010", - "contributions": [ - "doc" - ] - }, - { - "login": "klj890", - "name": "Tony Wang", - "avatar_url": "https://avatars3.githubusercontent.com/u/19380605?v=4", - "profile": "https://github.com/klj890", - "contributions": [ - "doc" - ] - }, - { - "login": "hlwanghl", - "name": "Hongliang Wang", - "avatar_url": "https://avatars3.githubusercontent.com/u/4861515?v=4", - "profile": "https://github.com/hlwanghl", - "contributions": [ - "code" - ] - }, - { - "login": "fafucoder", - "name": "dawn", - "avatar_url": "https://avatars0.githubusercontent.com/u/16442491?v=4", - "profile": "https://fafucoder.github.io/", - "contributions": [ - "code" - ] - }, - { - "login": "duanjiong", - "name": "Duan Jiong", - "avatar_url": "https://avatars1.githubusercontent.com/u/3678855?v=4", - "profile": "https://github.com/duanjiong", - "contributions": [ - "code" - ] - }, - { - "login": "calvinyv", - "name": "calvinyv", - "avatar_url": "https://avatars3.githubusercontent.com/u/28883416?v=4", - "profile": "https://github.com/calvinyv", - "contributions": [ - "doc" - ] - }, - { - "login": "benjaminhuo", - "name": "Benjamin Huo", - "avatar_url": "https://avatars2.githubusercontent.com/u/18525465?v=4", - "profile": "https://github.com/benjaminhuo", - "contributions": [ - "doc" - ] - }, - { - "login": "Sherlock113", - "name": "Sherlock113", - "avatar_url": "https://avatars2.githubusercontent.com/u/65327072?v=4", - "profile": "https://github.com/Sherlock113", - "contributions": [ - "doc" - ] - }, - { - "login": "Fuchange", - "name": "fu_changjie", - "avatar_url": "https://avatars1.githubusercontent.com/u/31716848?v=4", - "profile": "https://github.com/Fuchange", - "contributions": [ - "doc" - ] - }, - { - "login": "yuswift", - "name": "yuswift", - "avatar_url": "https://avatars1.githubusercontent.com/u/37265389?v=4", - "profile": "https://github.com/yuswift", - "contributions": [ - "code" - ] - }, - { - "login": "ruiyaoOps", - "name": "ruiyaoOps", - "avatar_url": "https://avatars.githubusercontent.com/u/35256376?v=4", - "profile": "https://github.com/ruiyaoOps", - "contributions": [ - "doc" - ] - }, - { - "login": "lxm", - "name": "LXM", - "avatar_url": "https://avatars.githubusercontent.com/u/1918195?v=4", - "profile": "http://www.luxingmin.com", - "contributions": [ - "doc" - ] - }, - { - "login": "sbhnet", - "name": "sbhnet", - "avatar_url": "https://avatars.githubusercontent.com/u/2368131?v=4", - "profile": "https://github.com/sbhnet", - "contributions": [ - "code" - ] - }, - { - "login": "misteruly", - "name": "misteruly", - "avatar_url": "https://avatars.githubusercontent.com/u/31399968?v=4", - "profile": "https://github.com/misteruly", - "contributions": [ - "code" - ] - }, - { - "login": "JohnNiang", - "name": "John Niang", - "avatar_url": "https://avatars.githubusercontent.com/u/16865714?v=4", - "profile": "https://johnniang.me", - "contributions": [ - "doc" - ] - }, - { - "login": "alimy", - "name": "Michael Li", - "avatar_url": "https://avatars.githubusercontent.com/u/10525842?v=4", - "profile": "https://alimy.me", - "contributions": [ - "code" - ] - }, - { - "login": "duguhaotian", - "name": "独孤昊天", - "avatar_url": "https://avatars.githubusercontent.com/u/3174621?v=4", - "profile": "https://github.com/duguhaotian", - "contributions": [ - "code" - ] - }, - { - "login": "lshmouse", - "name": "Liu Shaohui", - "avatar_url": "https://avatars.githubusercontent.com/u/118687?v=4", - "profile": "https://github.com/lshmouse", - "contributions": [ - "code" - ] - }, - { - "login": "24sama", - "name": "Leo Li", - "avatar_url": "https://avatars.githubusercontent.com/u/43993589?v=4", - "profile": "https://github.com/24sama", - "contributions": [ - "code" - ] - }, - { - "login": "RolandMa1986", - "name": "Roland", - "avatar_url": "https://avatars.githubusercontent.com/u/1720333?v=4", - "profile": "https://github.com/RolandMa1986", - "contributions": [ - "code" - ] - }, - { - "login": "vinsonzou", - "name": "Vinson Zou", - "avatar_url": "https://avatars.githubusercontent.com/u/2347587?v=4", - "profile": "https://ops.m114.org", - "contributions": [ - "doc" - ] - }, - { - "login": "tagGeeY", - "name": "tag_gee_y", - "avatar_url": "https://avatars.githubusercontent.com/u/35259969?v=4", - "profile": "https://github.com/tagGeeY", - "contributions": [ - "code" - ] - }, - { - "login": "liulangwa", - "name": "codebee", - "avatar_url": "https://avatars.githubusercontent.com/u/25916792?v=4", - "profile": "https://github.com/liulangwa", - "contributions": [ - "code" - ] - }, - { - "login": "TheApeMachine", - "name": "Daniel Owen van Dommelen", - "avatar_url": "https://avatars.githubusercontent.com/u/9572060?v=4", - "profile": "https://github.com/TheApeMachine", - "contributions": [ - "ideas" - ] - }, - { - "login": "Naidile-P-N", - "name": "Naidile P N", - "avatar_url": "https://avatars.githubusercontent.com/u/29476402?v=4", - "profile": "https://github.com/Naidile-P-N", - "contributions": [ - "code" - ] - }, - { - "login": "haiker2011", - "name": "Haiker Sun", - "avatar_url": "https://avatars.githubusercontent.com/u/8073429?v=4", - "profile": "https://github.com/haiker2011", - "contributions": [ - "code" - ] - }, - { - "login": "yj-cloud", - "name": "Jing Yu", - "avatar_url": "https://avatars.githubusercontent.com/u/19648473?v=4", - "profile": "https://github.com/yj-cloud", - "contributions": [ - "code" - ] - }, - { - "login": "chaunceyjiang", - "name": "Chauncey", - "avatar_url": "https://avatars.githubusercontent.com/u/17962021?v=4", - "profile": "https://github.com/chaunceyjiang", - "contributions": [ - "code" - ] - }, - { - "login": "tanguofu", - "name": "Tan Guofu", - "avatar_url": "https://avatars.githubusercontent.com/u/87045830?v=4", - "profile": "https://github.com/tanguofu", - "contributions": [ - "code" - ] - }, - { - "login": "lvillis", - "name": "lvillis", - "avatar_url": "https://avatars.githubusercontent.com/u/56720445?v=4", - "profile": "https://github.com/lvillis", - "contributions": [ - "doc" - ] - }, - { - "login": "vincenthe11", - "name": "Vincent He", - "avatar_url": "https://avatars.githubusercontent.com/u/8400716?v=4", - "profile": "https://github.com/vincenthe11", - "contributions": [ - "code" - ] - }, - { - "login": "tpiperatgod", - "name": "laminar", - "avatar_url": "https://avatars.githubusercontent.com/u/2360535?v=4", - "profile": "https://laminar.fun/", - "contributions": [ - "code" - ] - }, - { - "login": "cumirror", - "name": "tongjin", - "avatar_url": "https://avatars.githubusercontent.com/u/2455429?v=4", - "profile": "https://github.com/cumirror", - "contributions": [ - "code" - ] - }, - { - "login": "muzi502", - "name": "Reimu", - "avatar_url": "https://avatars.githubusercontent.com/u/42566386?v=4", - "profile": "http://k8s.li", - "contributions": [ - "code" - ] - }, - { - "login": "eltociear", - "name": "Ikko Ashimine", - "avatar_url": "https://avatars.githubusercontent.com/u/22633385?v=4", - "profile": "https://bandism.net/", - "contributions": [ - "doc" - ] - }, - { - "login": "yeya24", - "name": "Ben Ye", - "avatar_url": "https://avatars.githubusercontent.com/u/25150124?v=4", - "profile": "https://yeya24.github.io/", - "contributions": [ - "code" - ] - }, - { - "login": "yinheli", - "name": "yinheli", - "avatar_url": "https://avatars.githubusercontent.com/u/235094?v=4", - "profile": "https://github.com/yinheli", - "contributions": [ - "code" - ] - }, - { - "login": "hellocn9", - "name": "hellocn9", - "avatar_url": "https://avatars.githubusercontent.com/u/102210430?v=4", - "profile": "https://github.com/hellocn9", - "contributions": [ - "code" - ] - }, - { - "login": "brandan-schmitz", - "name": "Brandan Schmitz", - "avatar_url": "https://avatars.githubusercontent.com/u/6267549?v=4", - "profile": "https://github.com/brandan-schmitz", - "contributions": [ - "code" - ] - }, - { - "login": "yjqg6666", - "name": "yjqg6666", - "avatar_url": "https://avatars.githubusercontent.com/u/1879641?v=4", - "profile": "https://github.com/yjqg6666", - "contributions": [ - "doc", - "code" - ] - }, - { - "login": "zaunist", - "name": "失眠是真滴难受", - "avatar_url": "https://avatars.githubusercontent.com/u/38528079?v=4", - "profile": "https://github.com/zaunist", - "contributions": [ - "code" - ] - }, - { - "login": "mangoGoForward", - "name": "mango", - "avatar_url": "https://avatars.githubusercontent.com/u/35127166?v=4", - "profile": "https://github.com/mangoGoForward", - "contributions": [ - "review" - ] - }, - { - "login": "wenwutang1", - "name": "wenwutang", - "avatar_url": "https://avatars.githubusercontent.com/u/45817987?v=4", - "profile": "https://github.com/wenwutang1", - "contributions": [ - "code" - ] - }, - { - "login": "kuops", - "name": "Shiny Hou", - "avatar_url": "https://avatars.githubusercontent.com/u/18283256?v=4", - "profile": "http://kuops.com", - "contributions": [ - "code" - ] - }, - { - "login": "zhouqiu0103", - "name": "zhouqiu0103", - "avatar_url": "https://avatars.githubusercontent.com/u/108912268?v=4", - "profile": "https://github.com/zhouqiu0103", - "contributions": [ - "code" - ] - }, - { - "login": "77yu77", - "name": "77yu77", - "avatar_url": "https://avatars.githubusercontent.com/u/73932296?v=4", - "profile": "https://github.com/77yu77", - "contributions": [ - "code" - ] - }, - { - "login": "hzhhong", - "name": "hzhhong", - "avatar_url": "https://avatars.githubusercontent.com/u/83079531?v=4", - "profile": "https://github.com/hzhhong", - "contributions": [ - "code" - ] - }, - { - "login": "arugal", - "name": "zhang-wei", - "avatar_url": "https://avatars.githubusercontent.com/u/26432832?v=4", - "profile": "https://github.com/arugal", - "contributions": [ - "code" - ] - }, - { - "login": "xiaods", - "name": "Deshi Xiao", - "avatar_url": "https://avatars.githubusercontent.com/u/37678?v=4", - "profile": "https://twitter.com/xds2000", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "besscroft", - "name": "besscroft", - "avatar_url": "https://avatars.githubusercontent.com/u/33775809?v=4", - "profile": "https://besscroft.com", - "contributions": [ - "doc" - ] - }, - { - "login": "zhangzhiqiangcs", - "name": "张志强", - "avatar_url": "https://avatars.githubusercontent.com/u/8319897?v=4", - "profile": "https://github.com/zhangzhiqiangcs", - "contributions": [ - "code" - ] - }, - { - "login": "lwabish", - "name": "lwabish", - "avatar_url": "https://avatars.githubusercontent.com/u/7044019?v=4", - "profile": "https://github.com/lwabish", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "qyz87", - "name": "qyz87", - "avatar_url": "https://avatars.githubusercontent.com/u/36068894?v=4", - "profile": "https://github.com/qyz87", - "contributions": [ - "code" - ] - }, - { - "login": "fangzhengjin", - "name": "ZhengJin Fang", - "avatar_url": "https://avatars.githubusercontent.com/u/12680972?v=4", - "profile": "https://github.com/fangzhengjin", - "contributions": [ - "code" - ] - }, - { - "login": "ExerciseBook", - "name": "Eric_Lian", - "avatar_url": "https://avatars.githubusercontent.com/u/6327311?v=4", - "profile": "http://lhr.wiki", - "contributions": [ - "code" - ] - }, - { - "login": "nicognaW", - "name": "nicognaw", - "avatar_url": "https://avatars.githubusercontent.com/u/66731869?v=4", - "profile": "https://github.com/nicognaW", - "contributions": [ - "code" - ] - }, - { - "login": "deqingLv", - "name": "吕德庆", - "avatar_url": "https://avatars.githubusercontent.com/u/6064297?v=4", - "profile": "https://github.com/deqingLv", - "contributions": [ - "code" - ] - }, - { - "login": "littleplus", - "name": "littleplus", - "avatar_url": "https://avatars.githubusercontent.com/u/11694750?v=4", - "profile": "https://github.com/littleplus", - "contributions": [ - "code" - ] - }, - { - "login": "Nello-Angelo", - "name": "Konstantin", - "avatar_url": "https://avatars.githubusercontent.com/u/82488489?v=4", - "profile": "https://www.linkedin.com/in/%D0%BA%D0%BE%D0%BD%D1%81%D1%82%D0%B0%D0%BD%D1%82%D0%B8%D0%BD-%D0%B0%D0%BA%D0%B0%D0%BA%D0%B8%D0%B5%D0%B2-13130b1b4/", - "contributions": [ - "ideas" - ] - }, - { - "login": "kiragoo", - "name": "kiragoo", - "avatar_url": "https://avatars.githubusercontent.com/u/7400711?v=4", - "profile": "https://kiragoo.github.io", - "contributions": [ - "code" - ] - }, - { - "login": "jojotong", - "name": "jojotong", - "avatar_url": "https://avatars.githubusercontent.com/u/100849526?v=4", - "profile": "https://github.com/jojotong", - "contributions": [ - "code" - ] - }, - { - "login": "littleBlackHouse", - "name": "littleBlackHouse", - "avatar_url": "https://avatars.githubusercontent.com/u/54946465?v=4", - "profile": "https://github.com/littleBlackHouse", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "testwill", - "name": "guangwu", - "avatar_url": "https://avatars.githubusercontent.com/u/8717479?v=4", - "profile": "https://github.com/testwill", - "contributions": [ - "code", - "doc" - ] - }, - { - "login": "wongearl", - "name": "wongearl", - "avatar_url": "https://avatars.githubusercontent.com/u/36498442?v=4", - "profile": "https://github.com/wongearl", - "contributions": [ - "code" - ] - }, - { - "login": "wenwenxiong", - "name": "wenwenxiong", - "avatar_url": "https://avatars.githubusercontent.com/u/10548812?v=4", - "profile": "https://github.com/wenwenxiong", - "contributions": [ - "code" - ] - }, - { - "login": "BaiMeow", - "name": "柏喵Sakura", - "avatar_url": "https://avatars.githubusercontent.com/u/38121125?v=4", - "profile": "https://baimeow.cn/", - "contributions": [ - "code" - ] - }, - { - "login": "cuishuang", - "name": "cui fliter", - "avatar_url": "https://avatars.githubusercontent.com/u/15921519?v=4", - "profile": "https://dashen.tech", - "contributions": [ - "doc" - ] - }, - { - "login": "liuxu623", - "name": "刘旭", - "avatar_url": "https://avatars.githubusercontent.com/u/9653438?v=4", - "profile": "https://github.com/liuxu623", - "contributions": [ - "code" - ] - } - ], - "contributorsPerLine": 7, - "skipCi": true, - "commitType": "docs" -} diff --git a/.github/.stale.yaml b/.github/.stale.yaml deleted file mode 100644 index 8fa812eb..00000000 --- a/.github/.stale.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Number of days of inactivity before an issue becomes stale -daysUntilStale: 30 -# Number of days of inactivity before a stale issue is closed -daysUntilClose: 14 -# Issues with these labels will never be considered stale -exemptLabels: - - frozen -staleLabel: stale -# Comment to post when marking an issue as stale. Set to `false` to disable -markComment: > - This issue has been automatically marked as stale because it has not had - recent activity. It will be closed if no further activity occurs. Any further update will - cause the issue/pull request to no longer be considered stale. Thank you for your contributions. -# Comment to post when closing a stale issue. Set to `false` to disable -closeComment: > - This issue is being automatically closed due to inactivity. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml deleted file mode 100644 index d95b0554..00000000 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ /dev/null @@ -1,54 +0,0 @@ -name: Bug Report -description: File a bug report -labels: [bug] -body: - - type: markdown - id: preface - attributes: - value: | - Thanks for taking the time to fill out this bug report! - - type: markdown - id: environment - attributes: - value: "## Environment" - - type: input - id: version - validations: - required: true - attributes: - label: What is version of KubeKey has the issue? - description: "You can use the command to get `./kk version`" - - type: input - id: os - validations: - required: true - attributes: - label: What is your os environment? - description: "e.g. Ubuntu 16.04" - - type: textarea - id: config - attributes: - label: KubeKey config file - description: "If applicable, add the KubeKey config file content to help explain your problem." - render: yaml - - type: markdown - id: main - attributes: - value: "## Main" - - type: textarea - id: what - validations: - required: true - attributes: - label: A clear and concise description of what happend. - - type: textarea - id: logs - attributes: - label: "Relevant log output" - description: "Please copy and paste any relevant log output." - render: shell - - type: textarea - id: additional-information - attributes: - label: "Additional information" - description: "Add any other context about the information here (screenshots, video, etc.)." \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/custom.yaml b/.github/ISSUE_TEMPLATE/custom.yaml deleted file mode 100644 index 09e153b2..00000000 --- a/.github/ISSUE_TEMPLATE/custom.yaml +++ /dev/null @@ -1,5 +0,0 @@ -blank_issues_enabled: false -contact_links: - - name: Any other questions about KubeKey - url: https://kubesphere.slack.com/ - about: If you have any other questions, you can join our slack channel. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml deleted file mode 100644 index 34d4ff1a..00000000 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ /dev/null @@ -1,38 +0,0 @@ -name: Feature Request -description: File a feature request -body: - - type: markdown - id: preface - attributes: - value: "Thank you for submitting new features for KubeKey." - - type: markdown - id: environment - attributes: - value: "## Environment" - - type: input - id: version - attributes: - label: "Your current KubeKey version" - description: "You can use the command to get `./kk version`" - - type: markdown - id: main - attributes: - value: "## Main" - - type: textarea - id: description - attributes: - label: "Describe this feature" - validations: - required: true - - type: textarea - id: solution - attributes: - label: "Describe the solution you'd like" - description: "You can have a concise description of any alternative solutions or ideas you've considered." - validations: - required: true - - type: textarea - id: additional-information - attributes: - label: "Additional information" - description: "Add any other context about the information here (screenshots, video, etc.)." \ No newline at end of file diff --git a/.github/OWNERS b/.github/OWNERS deleted file mode 100644 index 45431bbf..00000000 --- a/.github/OWNERS +++ /dev/null @@ -1,18 +0,0 @@ -approvers: - - pixiake - - Forest-L - - rayzhou2017 - - LinuxSuRen - -reviewers: - - pixiake - - Forest-L - - rayzhou2017 - - zryfish - - shaowenchen - - benjaminhuo - - calvinyv - - FeynmanZhou - - huanggze - - wansir - - LinuxSuRen diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 97b13092..00000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,64 +0,0 @@ - - -### What type of PR is this? - - - -### What this PR does / why we need it: - -### Which issue(s) this PR fixes: - -Fixes # - -### Special notes for reviewers: -``` -``` - -### Does this PR introduced a user-facing change? - -```release-note - -``` - -### Additional documentation, usage docs, etc.: - -```docs - -``` diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index bcb619ec..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "daily" - pull-request-branch-name: - separator: "-" - - package-ecosystem: "gomod" - directory: "/" - schedule: - interval: "daily" - pull-request-branch-name: - separator: "-" diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml deleted file mode 100644 index f101c1f7..00000000 --- a/.github/release-drafter.yml +++ /dev/null @@ -1,41 +0,0 @@ -# Configuration for Release Drafter: https://github.com/toolmantim/release-drafter -name-template: 'v$NEXT_PATCH_VERSION 🌈' -tag-template: 'v$NEXT_PATCH_VERSION' -version-template: $MAJOR.$MINOR.$PATCH -# Emoji reference: https://gitmoji.carloscuesta.me/ -categories: - - title: '🚀 Features' - labels: - - 'feature' - - 'enhancement' - - 'kind/feature' - - title: '🐛 Bug Fixes' - labels: - - 'fix' - - 'bugfix' - - 'bug' - - 'regression' - - 'kind/bug' - - title: 📝 Documentation updates - labels: - - 'documentation' - - 'kind/documentation' - - title: 👻 Maintenance - labels: - - chore - - dependencies - - 'kind/cleanup' - - title: 🚦 Tests - labels: - - test - - tests -exclude-labels: - - reverted - - no-changelog - - skip-changelog - - invalid -change-template: '* $TITLE (#$NUMBER) @$AUTHOR' -template: | - ## What’s Changed - - $CHANGES diff --git a/.github/settings.yml b/.github/settings.yml deleted file mode 100644 index a412237e..00000000 --- a/.github/settings.yml +++ /dev/null @@ -1,68 +0,0 @@ -repository: - name: kubekey - description: "KubeKey provides a flexible, rapid and convenient way to install Kubernetes only, both Kubernetes and KubeSphere, and related cloud-native add-ons. It is also an efficient tool to scale and upgrade your cluster." - homepage: https://kubesphere.io/ - private: false - has_issues: true - has_wiki: false - has_downloads: false - default_branch: master - allow_squash_merge: true - allow_merge_commit: true - allow_rebase_merge: true -labels: - - name: newbie - color: abe7f4 - description: These're friendly issues for new comers - - name: bug - color: d73a4a - description: Something isn't working - - name: feature - color: ffc6a3 - - name: enhancement - color: a2eeef - description: New feature or request - - name: help wanted - color: 008672 - description: Extra attention is needed - - name: bugfix - color: 0412d6 - - name: regression - color: c5def5 - - name: documentation - color: 5ce05e - - name: Hacktoberfest - description: More details from https://hacktoberfest.digitalocean.com/ - color: 5ce05e - - name: test - color: c2c2fc - - name: chore - color: c2c2fc - - name: dependencies - color: 0366d6 - description: Pull requests that update a dependency file - - name: no-changelog - color: c2c2fc - - name: priority-high - color: D93F0B - - name: priority-medium - color: FBCA04 - - name: priority-low - color: 006B75 -branches: - - name: master - protection: - required_pull_request_reviews: - required_approving_review_count: 1 - dismiss_stale_reviews: true - require_code_owner_reviews: true - dismissal_restrictions: - users: [] - teams: [] - required_status_checks: - strict: true - contexts: [] - enforce_admins: false - restrictions: - users: [] - teams: [] diff --git a/.github/workflows/build-multiarch.yaml b/.github/workflows/build-multiarch.yaml deleted file mode 100644 index e0be29ad..00000000 --- a/.github/workflows/build-multiarch.yaml +++ /dev/null @@ -1,38 +0,0 @@ -name: BuildContainerImage - -on: - push: - tags: - - 'v3*' - -jobs: - build: - name: Build - runs-on: ubuntu-latest - env: - GO111MODULE: on - steps: - - name: Set env - run: echo "RELEASE_TAG=${GITHUB_REF:10}" >> $GITHUB_ENV - - - name: Check out code into the Go module directory - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - with: - platforms: all - - - name: Set up Docker buildx - uses: docker/setup-buildx-action@v2 - - - name: Build and push docker images - env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - if: github.event_name == 'push' - run: | - echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - make release-prod \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 5d480c5f..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: CI - -on: - push: - branches: - - 'master' - - 'release*' - tags: - - 'v*' - pull_request: - branches: - - 'master' - -jobs: - build: - name: Build - runs-on: ubuntu-latest - env: - GO111MODULE: on - steps: - - - name: Set up Go 1.19 - uses: actions/setup-go@v3 - with: - go-version: 1.19 - id: go - - - name: Check out code into the Go module directory - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - name: Downloading go dependencies - run: go mod tidy - - # If there are any diffs from goimports or go mod tidy, fail. - - name: Verify no changes from goimports and go mod tidy. - run: | - if [ -n "$(git status --porcelain)" ]; then - echo 'To fix this check, run "go mod tidy"' - git status # Show the files that failed to pass the check. - exit 1 - fi - - - name: Build command-line tool - run: | - make kk diff --git a/.github/workflows/gen-repository-iso.yaml b/.github/workflows/gen-repository-iso.yaml deleted file mode 100644 index 23428a9a..00000000 --- a/.github/workflows/gen-repository-iso.yaml +++ /dev/null @@ -1,77 +0,0 @@ ---- -name: gen-repository-iso -on: - push: - tags: - - 'v*' - workflow_dispatch: - -jobs: - build: - if: startsWith(github.ref, 'refs/tags/') - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - include: - - name: centos7-rpms - dockerfile: dockerfile.centos7 - - name: almalinux-9.0-rpms - dockerfile: dockerfile.almalinux90 - - name: debian10-debs - dockerfile: dockerfile.debian10 - - name: debian11-debs - dockerfile: dockerfile.debian11 - - name: ubuntu-16.04-debs - dockerfile: dockerfile.ubuntu1604 - - name: ubuntu-18.04-debs - dockerfile: dockerfile.ubuntu1804 - - name: ubuntu-20.04-debs - dockerfile: dockerfile.ubuntu2004 - - name: ubuntu-22.04-debs - dockerfile: dockerfile.ubuntu2204 - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Build iso image to local - uses: docker/build-push-action@v4 - with: - context: hack/gen-repository-iso - file: hack/gen-repository-iso/${{ matrix.dockerfile }} - platforms: linux/amd64,linux/arm64 - outputs: type=local,dest=./output - - - name: Prepare for upload iso - shell: bash - run: | - mv ./output/linux_amd64/*.iso ${{ matrix.name }}-amd64.iso - mv ./output/linux_arm64/*.iso ${{ matrix.name }}-arm64.iso - sha256sum *.iso > ${{ matrix.name }}.iso.sha256sum.txt - - - name: Wait for release workflow to finish - uses: lewagon/wait-on-check-action@v1.3.1 - with: - ref: ${{ github.ref }} - check-name: 'create draft release' - repo-token: ${{ secrets.GITHUB_TOKEN }} - wait-interval: 10 - allowed-conclusions: success - - - name: Release and upload packages - if: startsWith(github.ref, 'refs/tags/') - uses: softprops/action-gh-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - draft: true - files: | - ${{ matrix.name }}.iso.sha256sum.txt - ${{ matrix.name }}-amd64.iso - ${{ matrix.name }}-arm64.iso diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml deleted file mode 100644 index 04b4448c..00000000 --- a/.github/workflows/golangci-lint.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: golangci-lint - -on: - pull_request: - types: [opened, edited, synchronize, reopened] - -# Remove all permissions from GITHUB_TOKEN except metadata. -permissions: {} - -jobs: - golangci: - name: lint - runs-on: ubuntu-latest - strategy: - matrix: - working-directory: - - "" - - test - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version: 1.19 - - name: golangci-lint - uses: golangci/golangci-lint-action@v3.6.0 - with: - version: v1.50.1 - working-directory: ${{matrix.working-directory}} diff --git a/.github/workflows/kubernetes-auto-support.yaml b/.github/workflows/kubernetes-auto-support.yaml deleted file mode 100644 index 766365fe..00000000 --- a/.github/workflows/kubernetes-auto-support.yaml +++ /dev/null @@ -1,70 +0,0 @@ ---- -name: Kubernetes-Auto-Support - -on: - schedule: - - cron: '0 0 * * *' - -jobs: - update-kubernetes-version: - runs-on: ubuntu-latest - if: github.repository == 'kubesphere/kubekey' - steps: - - uses: actions/checkout@v3 - - - name: Set up Go 1.19 - uses: actions/setup-go@v3 - with: - go-version: 1.19 - id: go - - - name: install dependiencies - run: | - sudo apt update - sudo apt install skopeo -y - pip install natsort - wget https://attack-on-titan.gd2.qingstor.com/qsctl/v2.4.3/qsctl_v2.4.3_linux_amd64.tar.gz - tar -zxvf qsctl_v2.4.3_linux_amd64.tar.gz - mv qsctl_v2.4.3_linux_amd64 /usr/local/bin/qsctl - rm -rf qsctl_v2.4.3_linux_amd64.tar.gz - - - name: update components.json - id: get_new_version - run: | - chmod +x hack/auto-update-version.py - hack/auto-update-version.py - [ -f version.tmp ] && echo "UPDATE_VERSION=true" >> $GITHUB_OUTPUT || : - - - name: sync kubernetes - id: sync_kubernetes - run: | - for v in $(cat version.tmp) - do - KUBERNETES_VERSION=$v QSCTL_ACCESS_KEY_ID=${{ secrets.QSCTL_ACCESS_KEY_ID }} QSCTL_SECRET_ACCESS_KEY=${{secrets.QSCTL_SECRET_ACCESS_KEY}} DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }} ALIYUNCS_USERNAME=${{ secrets.ALIYUNCS_USERNAME }} ALIYUNCS_PASSWORD=${{ secrets.ALIYUNCS_PASSWORD }} bash hack/sync-components.sh - done - echo "NEW_VERSION=`cat version.tmp | tr '\n' ', '`" >> $GITHUB_OUTPUT - rm -rf qsctl-config.yaml - rm -rf version.tmp - - make kk - chmod +x bin/kk - echo "## Kubernetes Versions(amd64/arm64)" > docs/kubernetes-versions.md - echo "| Version | Supported |" >> docs/kubernetes-versions.md - echo "|----------|--------------------|" >> docs/kubernetes-versions.md - bin/kk version --show-supported-k8s | ( while read version; do echo "| $version | :white_check_mark: |" >> docs/kubernetes-versions.md; done ) - - if: steps.get_new_version.outputs.UPDATE_VERSION == 'true' - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v4 - with: - commit-message: Add new kubernetes version - committer: GitHub - signoff: false - branch: new_version - delete-branch: true - title: 'Add new kubernetes version' - body: | - Add kubernetes version: ${{ steps.sync_kubernetes.outputs.NEW_VERSION }} - - if: steps.get_new_version.outputs.UPDATE_VERSION == 'true' diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml deleted file mode 100644 index 17fdb961..00000000 --- a/.github/workflows/release-drafter.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Release Drafter - -on: - push: - branches: - - master - -jobs: - update_release_draft: - runs-on: ubuntu-latest - steps: - - uses: release-drafter/release-drafter@v5 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml deleted file mode 100644 index 19ac80a0..00000000 --- a/.github/workflows/release.yaml +++ /dev/null @@ -1,46 +0,0 @@ -name: Release - -on: - push: - tags: - - '*' - -jobs: - build: - name: create draft release - runs-on: ubuntu-latest - steps: - - name: Set env - run: echo "RELEASE_TAG=${GITHUB_REF:10}" >> $GITHUB_ENV - - name: checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Install go - uses: actions/setup-go@v3 - with: - go-version: '^1.19' - - name: generate release artifacts - run: | - make release - - name: Release - uses: softprops/action-gh-release@v1 - with: - draft: true - files: out/* - - - name: Get Version - id: get_version - run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT - - name: Synchronize artifacts to OSS - run: | - rm -rf qsctl_v2.4.3_linux_amd64.tar.gz - wget https://attack-on-titan.gd2.qingstor.com/qsctl/v2.4.3/qsctl_v2.4.3_linux_amd64.tar.gz - tar -zxvf qsctl_v2.4.3_linux_amd64.tar.gz - rm -rf qsctl_v2.4.3_linux_amd64.tar.gz - mv qsctl_v2.4.3_linux_amd64 /usr/local/bin/qsctl - echo "access_key_id: ${{secrets.KS_QSCTL_ACCESS_KEY_ID}}" > qsctl-config.yaml - echo "secret_access_key: ${{ secrets.KS_QSCTL_SECRET_ACCESS_KEY }}" >> qsctl-config.yaml - qsctl cp out/kubekey-${{ steps.get_version.outputs.VERSION }}-linux-amd64.tar.gz qs://kubernetes/kubekey/releases/download/${{ steps.get_version.outputs.VERSION }}/kubekey-${{ steps.get_version.outputs.VERSION }}-linux-amd64.tar.gz -c qsctl-config.yaml - qsctl cp out/kubekey-${{ steps.get_version.outputs.VERSION }}-linux-arm64.tar.gz qs://kubernetes/kubekey/releases/download/${{ steps.get_version.outputs.VERSION }}/kubekey-${{ steps.get_version.outputs.VERSION }}-linux-arm64.tar.gz -c qsctl-config.yaml - rm -rf qsctl-config.yaml diff --git a/.gitignore b/.gitignore deleted file mode 100644 index b7fb3b29..00000000 --- a/.gitignore +++ /dev/null @@ -1,38 +0,0 @@ -# Binaries for programs and plugins -*.exe -*.dll -*.so -*.dylib -*.tmp -bin -hack/tools/bin - -# Test binary, build with `go test -c` -*.test - -# E2E test templates -test/e2e/data/infrastructure-kubekey/v1beta1/cluster-template*.yaml - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# IntelliJ -.idea/ -*.iml - -# Vscode files -.vscode - -# rbac and manager config for example provider -manager_image_patch.yaml-e -manager_pull_policy.yaml-e - -# Sample config files auto-generated by kubebuilder -config/samples - -# test results -_artifacts - -# Used during parts of the build process. Files _should_ get cleaned up automatically. -# This is also a good location for any temporary manfiests used during development -tmp \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml deleted file mode 100644 index 51174998..00000000 --- a/.golangci.yml +++ /dev/null @@ -1,288 +0,0 @@ -linters: - disable-all: true - enable: - - asciicheck - - bodyclose - - containedctx - - deadcode - - depguard - - dogsled - - errcheck - - exportloopref - - gci - - goconst - - gocritic - - gofmt - - goimports - - goprintffuncname - - gosec - - gosimple - - govet - - importas - - ineffassign - - misspell - - nakedret - - nilerr - - noctx - - nolintlint - - prealloc - - predeclared - - revive - - rowserrcheck - - staticcheck - - structcheck - - stylecheck - - thelper - - typecheck - - unconvert - - unparam - - unused - - varcheck - - whitespace - -linters-settings: - godot: - # declarations - for top level declaration comments (default); - # toplevel - for top level comments; - # all - for all comments. - scope: toplevel - exclude: - - '^ \+.*' - - '^ ANCHOR.*' - ifshort: - # Maximum length of variable declaration measured in number of characters, after which linter won't suggest using short syntax. - max-decl-chars: 50 - gci: - local-prefixes: "github.com/kubesphere/kubekey" - importas: - no-unaliased: true - alias: - # Kubernetes - - pkg: k8s.io/api/core/v1 - alias: corev1 - - pkg: k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 - alias: apiextensionsv1 - - pkg: k8s.io/apimachinery/pkg/apis/meta/v1 - alias: metav1 - - pkg: k8s.io/apimachinery/pkg/api/errors - alias: apierrors - - pkg: k8s.io/apimachinery/pkg/util/errors - alias: kerrors - # Controller Runtime - - pkg: sigs.k8s.io/controller-runtime - alias: ctrl - # CABPK - - pkg: sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3 - alias: bootstrapv1alpha3 - - pkg: sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha4 - alias: bootstrapv1alpha4 - - pkg: sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1 - alias: bootstrapv1 - # KCP - - pkg: sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3 - alias: controlplanev1alpha3 - - pkg: sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha4 - alias: controlplanev1alpha4 - - pkg: sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1 - alias: controlplanev1 - # CAPI - - pkg: sigs.k8s.io/cluster-api/api/v1alpha3 - alias: clusterv1alpha3 - - pkg: sigs.k8s.io/cluster-api/api/v1alpha4 - alias: clusterv1alpha4 - - pkg: sigs.k8s.io/cluster-api/api/v1beta1 - alias: clusterv1 - # CAPI exp - - pkg: sigs.k8s.io/cluster-api/exp/api/v1alpha3 - alias: expv1alpha3 - - pkg: sigs.k8s.io/cluster-api/exp/api/v1alpha4 - alias: expv1alpha4 - - pkg: sigs.k8s.io/cluster-api/exp/api/v1beta1 - alias: expv1 - # CAPI exp addons - - pkg: sigs.k8s.io/cluster-api/exp/addons/api/v1alpha3 - alias: addonsv1alpha3 - - pkg: sigs.k8s.io/cluster-api/exp/addons/api/v1alpha4 - alias: addonsv1alpha4 - - pkg: sigs.k8s.io/cluster-api/exp/addons/api/v1beta1 - alias: addonsv1 - # CAPI exp runtime - - pkg: sigs.k8s.io/cluster-api/exp/runtime/api/v1alpha1 - alias: runtimev1 - - pkg: sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1 - alias: runtimehooksv1 - - pkg: sigs.k8s.io/cluster-api/exp/runtime/controllers - alias: runtimecontrollers - - pkg: sigs.k8s.io/cluster-api/exp/runtime/catalog - alias: runtimecatalog - - pkg: sigs.k8s.io/cluster-api/internal/runtime/client - alias: runtimeclient - - pkg: sigs.k8s.io/cluster-api/internal/runtime/registry - alias: runtimeregistry - - pkg: sigs.k8s.io/cluster-api/internal/webhooks/runtime - alias: runtimewebhooks - # CAPKK - - pkg: github.com/kubesphere/kubekey/v3/api/v1beta1 - alias: infrav1 - - pkg: github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1 - alias: infrabootstrapv1 - - pkg: github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1 - alias: infracontrolplanev1 - nolintlint: - allow-unused: false - allow-leading-space: false - require-specific: true - revive: - rules: - # The following rules are recommended https://github.com/mgechev/revive#recommended-configuration - - name: blank-imports - - name: context-as-argument - - name: context-keys-type - - name: dot-imports - - name: error-return - - name: error-strings - - name: error-naming - - name: exported - #- name: if-return # TODO This is a recommended rule with many findings which may require it's own pr. - - name: increment-decrement - - name: var-naming - - name: var-declaration - - name: package-comments - - name: range - - name: receiver-naming - - name: time-naming - - name: unexported-return - - name: indent-error-flow - - name: errorf - - name: empty-block - - name: superfluous-else - #- name: unused-parameter # TODO This is a recommended rule with many findings which may require it's own pr. - - name: unreachable-code - - name: redefines-builtin-id - # - # Rules in addition to the recommended configuration above. - # - - name: bool-literal-in-expr - - name: constant-logical-expr - gosec: - excludes: - - G307 # Deferring unsafe method "Close" on type "\*os.File" - - G108 # Profiling endpoint is automatically exposed on /debug/pprof - gocritic: - enabled-tags: - - experimental - disabled-checks: - - appendAssign - - dupImport # https://github.com/go-critic/go-critic/issues/845 - - evalOrder - - ifElseChain - - octalLiteral - - regexpSimplify - - sloppyReassign - - truncateCmp - - typeDefFirst - - unnamedResult - - unnecessaryDefer - - whyNoLint - - wrapperFunc - - commentFormatting - - filepathJoin - - rangeValCopy - - hugeParam -issues: - max-same-issues: 0 - max-issues-per-linter: 0 - # We are disabling default golangci exclusions because we want to help reviewers to focus on reviewing the most relevant - # changes in PRs and avoid nitpicking. - exclude-use-default: false - exclude-rules: - - linters: - - revive - text: "exported: exported method .*\\.(Reconcile|SetupWithManager|SetupWebhookWithManager) should have comment or be unexported" - - linters: - - errcheck - text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked - # Exclude revive's exported for certain packages and code, e.g. tests and fake. - - linters: - - revive - text: exported (method|function|type|const) (.+) should have comment or be unexported - source: (func|type).*Fake.* - - linters: - - revive - text: exported (method|function|type|const) (.+) should have comment or be unexported - path: fake_\.go - - linters: - - revive - text: exported (method|function|type|const) (.+) should have comment or be unexported - path: .*test/(providers|framework|e2e).*.go - - linters: - - errcheck - text: Error return value is not checked - path: _test\.go - - linters: - - errcheck - text: Error return value of (.+) is not checked - path: _test\.go - - linters: - - gosec - text: "G108: Profiling endpoint is automatically exposed on /debug/pprof" - - linters: - - godot - text: "Comment should end in a period" - path: "(.*)/(v1beta1|v1beta2)/(.*)types.go" - - linters: - - errcheck - text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked - # With Go 1.16, the new embed directive can be used with an un-named import, - # revive (previously, golint) only allows these to be imported in a main.go, which wouldn't work for us. - # This directive allows the embed package to be imported with an underscore everywhere. - - linters: - - revive - source: _ "embed" - # This directive allows the variable in defaults.go files to have underscore - - linters: - - revive - text: "var-naming: don't use underscores in Go names; func (.+) should be (.+)" - path: .*/defaults.go - # Disable unparam "always receives" which might not be really - # useful when building libraries. - - linters: - - unparam - text: always receives - # Dot imports for gomega or ginkgo are allowed - # within test files. - - path: _test\.go - text: should not use dot imports - - path: (framework|e2e)/.*.go - text: should not use dot imports - - path: _test\.go - text: cyclomatic complexity - - linters: - - unparam - text: (.+) - (`t`|`g`) is unused - - path: _test\.go - text: cyclomatic complexity - # Append should be able to assign to a different var/slice. - - linters: - - gocritic - text: "appendAssign: append result not assigned to the same slice" - # We don't care about defer in for loops in test files. - - linters: - - gocritic - text: "deferInLoop: Possible resource leak, 'defer' is called in the 'for' loop" - path: _test\.go - -run: - timeout: 10m - go: "1.19" - build-tags: - - tools - - e2e - - containers_image_openpgp - - exclude_graphdriver_devicemapper - - exclude_graphdriver_btrfs - skip-files: - - "zz_generated.*\\.go$" - - "vendored_openapi\\.go$" - - "cmd" - allow-parallel-runners: true diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md deleted file mode 100644 index cb04f230..00000000 --- a/CONTRIBUTORS.md +++ /dev/null @@ -1,122 +0,0 @@ -### Sincere gratitude goes to the following people for their contributions to KubeKey - -Contributions of any kind are welcome! Thanks goes to these wonderful contributors, they made our project grow fast. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
pixiake
pixiake
💻 📖		Forest
Forest
💻 📖		rayzhou2017
rayzhou2017
💻 📖		shaowenchen
shaowenchen
💻 📖		Zhao Xiaojie
Zhao Xiaojie
💻 📖		Zack Zhang
Zack Zhang
💻		Akhil Mohan
Akhil Mohan
💻
pengfei
pengfei
📖		min zhang
min zhang
💻 📖		zgldh
zgldh
💻		xrjk
xrjk
💻		yonghongshi
yonghongshi
💻		Honglei
Honglei
📖		liucy1983
liucy1983
💻
Lien
Lien
📖		Tony Wang
Tony Wang
📖		Hongliang Wang
Hongliang Wang
💻		dawn
dawn
💻		Duan Jiong
Duan Jiong
💻		calvinyv
calvinyv
📖		Benjamin Huo
Benjamin Huo
📖
Sherlock113
Sherlock113
📖		fu_changjie
fu_changjie
📖		yuswift
yuswift
💻		ruiyaoOps
ruiyaoOps
📖		LXM
LXM
📖		sbhnet
sbhnet
💻		misteruly
misteruly
💻
John Niang
John Niang
📖		Michael Li
Michael Li
💻		独孤昊天
独孤昊天
💻		Liu Shaohui
Liu Shaohui
💻		Leo Li
Leo Li
💻		Roland
Roland
💻		Vinson Zou
Vinson Zou
📖
tag_gee_y
tag_gee_y
💻		codebee
codebee
💻		Daniel Owen van Dommelen
Daniel Owen van Dommelen
🤔		Naidile P N
Naidile P N
💻		Haiker Sun
Haiker Sun
💻		Jing Yu
Jing Yu
💻		Chauncey
Chauncey
💻
Tan Guofu
Tan Guofu
💻		lvillis
lvillis
📖		Vincent He
Vincent He
💻		laminar
laminar
💻		tongjin
tongjin
💻		Reimu
Reimu
💻		Ikko Ashimine
Ikko Ashimine
📖
Ben Ye
Ben Ye
💻		yinheli
yinheli
💻		hellocn9
hellocn9
💻		Brandan Schmitz
Brandan Schmitz
💻		yjqg6666
yjqg6666
📖 💻		失眠是真滴难受
失眠是真滴难受
💻		mango
mango
👀
wenwutang
wenwutang
💻		Shiny Hou
Shiny Hou
💻		zhouqiu0103
zhouqiu0103
💻		77yu77
77yu77
💻		hzhhong
hzhhong
💻		zhang-wei
zhang-wei
💻		Deshi Xiao
Deshi Xiao
💻 📖
besscroft
besscroft
📖		张志强
张志强
💻		lwabish
lwabish
💻 📖		qyz87
qyz87
💻		ZhengJin Fang
ZhengJin Fang
💻		Eric_Lian
Eric_Lian
💻		nicognaw
nicognaw
💻
吕德庆
吕德庆
💻		littleplus
littleplus
💻		Konstantin
Konstantin
🤔		kiragoo
kiragoo
💻		jojotong
jojotong
💻		littleBlackHouse
littleBlackHouse
💻 📖		guangwu
guangwu
💻 📖
wongearl
wongearl
💻		wenwenxiong
wenwenxiong
💻		柏喵Sakura
柏喵Sakura
💻		cui fliter
cui fliter
📖		刘旭
刘旭
💻
- - - - - diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 7584f3b7..00000000 --- a/Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ -# Build architecture -ARG ARCH - -# Download dependencies -FROM alpine:3.11 as base_os_context - - -ENV OUTDIR=/out -RUN mkdir -p ${OUTDIR}/usr/local/bin/ - -WORKDIR /tmp - -RUN apk add --no-cache ca-certificates - -# Build the manager binary -FROM golang:1.19 as builder - -# Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy -ARG goproxy=https://goproxy.cn,direct -ENV GOPROXY=$goproxy - -WORKDIR /workspace - -COPY go.mod go.mod -COPY go.sum go.sum - -# Cache deps before building and copying source so that we don't need to re-download as much -# and so that source changes don't invalidate our downloaded layer -RUN --mount=type=cache,target=/go/pkg/mod \ - go mod download - -# Copy the go source -COPY ./ ./ - -# Cache the go build into the the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls -RUN --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg/mod \ - go build . - -# Build -ARG package=. -ARG ARCH -ARG LDFLAGS - -# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder -RUN --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg/mod \ - CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \ - go build -ldflags "${LDFLAGS}" \ - -o manager ${package} - -FROM --platform=${ARCH} alpine:3.16 - -WORKDIR / - -RUN mkdir -p /var/lib/kubekey/rootfs - -COPY --from=base_os_context /out/ / -COPY --from=builder /workspace/manager . - -ENTRYPOINT ["/manager"] diff --git a/LICENSE b/LICENSE deleted file mode 100644 index cd92c18d..00000000 --- a/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2018-2020 KubeSphere Authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/Makefile b/Makefile deleted file mode 100644 index 7d8d0767..00000000 --- a/Makefile +++ /dev/null @@ -1,601 +0,0 @@ -# Ensure Make is run with bash shell as some syntax below is bash-specific -SHELL:=/usr/bin/env bash - -.DEFAULT_GOAL:=help - -# -# Go. -# -GO_VERSION ?= 1.19.2 -GO_CONTAINER_IMAGE ?= docker.io/library/golang:$(GO_VERSION) - -# Use GOPROXY environment variable if set -GOPROXY := $(shell go env GOPROXY) -ifeq ($(GOPROXY),) -GOPROXY := https://goproxy.cn,direct -endif -export GOPROXY - -# Active module mode, as we use go modules to manage dependencies -export GO111MODULE=on - -# This option is for running docker manifest command -export DOCKER_CLI_EXPERIMENTAL := enabled - -# -# Directories. -# -# Full directory of where the Makefile resides -ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) -EXP_DIR := exp -BIN_DIR := bin -TEST_DIR := test -TOOLS_DIR := hack/tools -TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/$(BIN_DIR)) -E2E_FRAMEWORK_DIR := $(TEST_DIR)/framework -GO_INSTALL := ./scripts/go_install.sh - -export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) - -# -# Binaries. -# -# Note: Need to use abspath so we can invoke these from subdirectories -KUSTOMIZE_VER := v4.5.2 -KUSTOMIZE_BIN := kustomize -KUSTOMIZE := $(abspath $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER)) -KUSTOMIZE_PKG := sigs.k8s.io/kustomize/kustomize/v4 - -SETUP_ENVTEST_VER := v0.0.0-20211110210527-619e6b92dab9 -SETUP_ENVTEST_BIN := setup-envtest -SETUP_ENVTEST := $(abspath $(TOOLS_BIN_DIR)/$(SETUP_ENVTEST_BIN)-$(SETUP_ENVTEST_VER)) -SETUP_ENVTEST_PKG := sigs.k8s.io/controller-runtime/tools/setup-envtest - -CONTROLLER_GEN_VER := v0.9.1 -CONTROLLER_GEN_BIN := controller-gen -CONTROLLER_GEN := $(abspath $(TOOLS_BIN_DIR)/$(CONTROLLER_GEN_BIN)-$(CONTROLLER_GEN_VER)) -CONTROLLER_GEN_PKG := sigs.k8s.io/controller-tools/cmd/controller-gen - -GOTESTSUM_VER := v1.6.4 -GOTESTSUM_BIN := gotestsum -GOTESTSUM := $(abspath $(TOOLS_BIN_DIR)/$(GOTESTSUM_BIN)-$(GOTESTSUM_VER)) -GOTESTSUM_PKG := gotest.tools/gotestsum - -HADOLINT_VER := v2.10.0 -HADOLINT_FAILURE_THRESHOLD = warning - -GOLANGCI_LINT_BIN := golangci-lint -GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/$(GOLANGCI_LINT_BIN)) - -# Define Docker related variables. Releases should modify and double check these vars. -REGISTRY ?= docker.io/kubespheredev -PROD_REGISTRY ?= docker.io/kubesphere - -# capkk -CAPKK_IMAGE_NAME ?= capkk-controller -CAPKK_CONTROLLER_IMG ?= $(REGISTRY)/$(CAPKK_IMAGE_NAME) - -# bootstrap -K3S_BOOTSTRAP_IMAGE_NAME ?= k3s-bootstrap-controller -K3S_BOOTSTRAP_CONTROLLER_IMG ?= $(REGISTRY)/$(K3S_BOOTSTRAP_IMAGE_NAME) - -# control plane -K3S_CONTROL_PLANE_IMAGE_NAME ?= k3s-control-plane-controller -K3S_CONTROL_PLANE_CONTROLLER_IMG ?= $(REGISTRY)/$(K3S_CONTROL_PLANE_IMAGE_NAME) - -# It is set by Prow GIT_TAG, a git-based tag of the form vYYYYMMDD-hash, e.g., v20210120-v0.3.10-308-gc61521971 - -TAG ?= dev -ARCH ?= $(shell go env GOARCH) -ALL_ARCH = amd64 arm arm64 ppc64le s390x - -# Allow overriding the imagePullPolicy -PULL_POLICY ?= Always - -# Hosts running SELinux need :z added to volume mounts -SELINUX_ENABLED := $(shell cat /sys/fs/selinux/enforce 2> /dev/null || echo 0) - -ifeq ($(SELINUX_ENABLED),1) - DOCKER_VOL_OPTS?=:z -endif - -# Set build time variables including version details -LDFLAGS := $(shell hack/version.sh) - -# Set kk build tags -BUILDTAGS = exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp - -.PHONY: all -all: test managers - -.PHONY: help -help: ## Display this help. - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n\nTargets:\n"} /^[0-9A-Za-z_-]+:.*?##/ { printf " \033[36m%-45s\033[0m %s\n", $$1, $$2 } /^\$$\([0-9A-Za-z_-]+\):.*?##/ { gsub("_","-", $$1); printf " \033[36m%-45s\033[0m %s\n", tolower(substr($$1, 3, length($$1)-7)), $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -## -------------------------------------- -## Generate / Manifests -## -------------------------------------- - -##@ generate: - -ALL_GENERATE_MODULES = capkk k3s-bootstrap k3s-control-plane - -.PHONY: generate -generate: ## Run all generate-manifests-*, generate-go-deepcopy-* targets - $(MAKE) generate-modules generate-manifests generate-go-deepcopy - -.PHONY: generate-manifests -generate-manifests: ## Run all generate-manifest-* targets - $(MAKE) $(addprefix generate-manifests-,$(ALL_GENERATE_MODULES)) - -.PHONY: generate-manifests-capkk -generate-manifests-capkk: $(CONTROLLER_GEN) $(KUSTOMIZE) ## Generate manifests e.g. CRD, RBAC etc. for core - $(MAKE) clean-generated-yaml SRC_DIRS="./config/crd/bases" - $(CONTROLLER_GEN) \ - paths=./api/... \ - paths=./controllers/... \ - crd:crdVersions=v1 \ - rbac:roleName=manager-role \ - output:crd:dir=./config/crd/bases \ - output:webhook:dir=./config/webhook \ - webhook - -.PHONY: generate-manifests-k3s-bootstrap -generate-manifests-k3s-bootstrap: $(CONTROLLER_GEN) $(KUSTOMIZE) ## Generate manifests e.g. CRD, RBAC etc. for core - $(MAKE) clean-generated-yaml SRC_DIRS="./bootstrap/k3s/config/crd/bases" - $(CONTROLLER_GEN) \ - paths=./bootstrap/k3s/api/... \ - paths=./bootstrap/k3s/controllers/... \ - crd:crdVersions=v1 \ - rbac:roleName=manager-role \ - output:crd:dir=./bootstrap/k3s/config/crd/bases \ - output:rbac:dir=./bootstrap/k3s/config/rbac \ - output:webhook:dir=./bootstrap/k3s/config/webhook \ - webhook - -.PHONY: generate-manifests-k3s-control-plane -generate-manifests-k3s-control-plane: $(CONTROLLER_GEN) $(KUSTOMIZE) ## Generate manifests e.g. CRD, RBAC etc. for core - $(MAKE) clean-generated-yaml SRC_DIRS="./controlplane/k3s/config/crd/bases" - $(CONTROLLER_GEN) \ - paths=./controlplane/k3s/api/... \ - paths=./controlplane/k3s/controllers/... \ - crd:crdVersions=v1 \ - rbac:roleName=manager-role \ - output:crd:dir=./controlplane/k3s/config/crd/bases \ - output:rbac:dir=./controlplane/k3s/config/rbac \ - output:webhook:dir=./controlplane/k3s/config/webhook \ - webhook - -.PHONY: generate-go-deepcopy -generate-go-deepcopy: ## Run all generate-go-deepcopy-* targets - $(MAKE) $(addprefix generate-go-deepcopy-,$(ALL_GENERATE_MODULES)) - -.PHONY: generate-go-deepcopy-capkk -generate-go-deepcopy-capkk: $(CONTROLLER_GEN) ## Generate deepcopy go code for capkk - $(MAKE) clean-generated-deepcopy SRC_DIRS="./api" - $(CONTROLLER_GEN) \ - object:headerFile=./hack/boilerplate.go.txt \ - paths=./api/... \ - -.PHONY: generate-go-deepcopy-k3s-bootstrap -generate-go-deepcopy-k3s-bootstrap: $(CONTROLLER_GEN) ## Generate deepcopy go code for k3s-bootstrap - $(MAKE) clean-generated-deepcopy SRC_DIRS="./bootstrap/k3s/api" - $(CONTROLLER_GEN) \ - object:headerFile=./hack/boilerplate.go.txt \ - paths=./bootstrap/k3s/api/... \ - -.PHONY: generate-go-deepcopy-k3s-control-plane -generate-go-deepcopy-k3s-control-plane: $(CONTROLLER_GEN) ## Generate deepcopy go code for k3s-control-plane - $(MAKE) clean-generated-deepcopy SRC_DIRS="./controlplane/k3s/api" - $(CONTROLLER_GEN) \ - object:headerFile=./hack/boilerplate.go.txt \ - paths=./controlplane/k3s/api/... \ - -.PHONY: generate-modules -generate-modules: ## Run go mod tidy to ensure modules are up to date - go mod tidy - -## -------------------------------------- -## Lint / Verify -## -------------------------------------- - -##@ lint and verify: - -.PHONY: lint -lint: $(GOLANGCI_LINT) ## Lint the codebase - $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS) - cd $(TEST_DIR); $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS) - cd $(TOOLS_DIR); $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS) - ./scripts/ci-lint-dockerfiles.sh $(HADOLINT_VER) $(HADOLINT_FAILURE_THRESHOLD) - -.PHONY: lint-dockerfiles -lint-dockerfiles: - ./scripts/ci-lint-dockerfiles.sh $(HADOLINT_VER) $(HADOLINT_FAILURE_THRESHOLD) - -.PHONY: verify -verify: $(addprefix verify-,$(ALL_VERIFY_CHECKS)) lint-dockerfiles ## Run all verify-* targets - -.PHONY: verify-modules -verify-modules: generate-modules ## Verify go modules are up to date - @if !(git diff --quiet HEAD -- go.sum go.mod $(TOOLS_DIR)/go.mod $(TOOLS_DIR)/go.sum $(TEST_DIR)/go.mod $(TEST_DIR)/go.sum); then \ - git diff; \ - echo "go module files are out of date"; exit 1; \ - fi - @if (find . -name 'go.mod' | xargs -n1 grep -q -i 'k8s.io/client-go.*+incompatible'); then \ - find . -name "go.mod" -exec grep -i 'k8s.io/client-go.*+incompatible' {} \; -print; \ - echo "go module contains an incompatible client-go version"; exit 1; \ - fi - -.PHONY: verify-gen -verify-gen: generate ## Verify go generated files are up to date - @if !(git diff --quiet HEAD); then \ - git diff; \ - echo "generated files are out of date, run make generate"; exit 1; \ - fi - -## -------------------------------------- -## Binaries -## -------------------------------------- - -##@ build: - -.PHONY: kk -kk: - CGO_ENABLED=0 go build -trimpath -tags "$(BUILDTAGS)" -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/kk github.com/kubesphere/kubekey/v3/cmd/kk; - -ALL_MANAGERS = capkk k3s-bootstrap k3s-control-plane - -.PHONY: managers -managers: $(addprefix manager-,$(ALL_MANAGERS)) ## Run all manager-* targets - -.PHONY: manager-capkk -manager-capkk: ## Build the capkk manager binary into the ./bin folder - go build -trimpath -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/manager github.com/kubesphere/kubekey/v3 - -.PHONY: manager-k3s-bootstrap -manager-k3s-bootstrap: ## Build the k3s bootstrap manager binary into the ./bin folder - go build -trimpath -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/k3s-bootstrap-manager github.com/kubesphere/kubekey/v3/bootstrap/k3s - -.PHONY: manager-k3s-control-plane -manager-k3s-control-plane: ## Build the k3s control plane manager binary into the ./bin folder - go build -trimpath -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/k3s-control-plane-manager github.com/kubesphere/kubekey/v3/controlplane/k3s - -.PHONY: docker-pull-prerequisites -docker-pull-prerequisites: - docker pull docker.io/docker/dockerfile:1.4 - docker pull $(GO_CONTAINER_IMAGE) - -.PHONY: docker-build-all -docker-build-all: $(addprefix docker-build-,$(ALL_ARCH)) ## Build docker images for all architectures - -docker-build-%: - $(MAKE) ARCH=$* docker-build - -ALL_DOCKER_BUILD = capkk k3s-bootstrap k3s-control-plane - -.PHONY: docker-build -docker-build: docker-pull-prerequisites ## Run docker-build-* targets for all providers - $(MAKE) ARCH=$(ARCH) $(addprefix docker-build-,$(ALL_DOCKER_BUILD)) - -.PHONY: docker-build-capkk -docker-build-capkk: ## Build the docker image for capkk - DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(CAPKK_CONTROLLER_IMG)-$(ARCH):$(TAG) - -.PHONY: docker-build-k3s-bootstrap -docker-build-k3s-bootstrap: ## Build the docker image for k3s bootstrap controller manager - DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg package=./bootstrap/k3s --build-arg ldflags="$(LDFLAGS)" . -t $(K3S_BOOTSTRAP_CONTROLLER_IMG)-$(ARCH):$(TAG) - -.PHONY: docker-build-k3s-control-plane -docker-build-k3s-control-plane: ## Build the docker image for k3s control plane controller manager - DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg package=./controlplane/k3s --build-arg ldflags="$(LDFLAGS)" . -t $(K3S_CONTROL_PLANE_CONTROLLER_IMG)-$(ARCH):$(TAG) - -.PHONY: docker-build-e2e -docker-build-e2e: ## Build the docker image for capkk - $(MAKE) docker-build REGISTRY=docker.io/kubespheredev PULL_POLICY=IfNotPresent TAG=e2e - -## -------------------------------------- -## Deployment -## -------------------------------------- - -##@ deployment - -ifndef ignore-not-found - ignore-not-found = false -endif - -.PHONY: install -install: generate $(KUSTOMIZE) ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | kubectl apply -f - - -.PHONY: uninstall -uninstall: generate $(KUSTOMIZE) ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - - -.PHONY: deploy -deploy: generate $(KUSTOMIZE) ## Deploy controller to the K8s cluster specified in ~/.kube/config. - $(MAKE) set-manifest-image \ - MANIFEST_IMG=$(REGISTRY)/$(CAPKK_IMAGE_NAME)-$(ARCH) MANIFEST_TAG=$(TAG) \ - TARGET_RESOURCE="./config/default/manager_image_patch.yaml" - cd config/manager - $(KUSTOMIZE) build config/default | kubectl apply -f - - -.PHONY: undeploy -undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - - -## -------------------------------------- -## Testing -## -------------------------------------- - -##@ test: - -ARTIFACTS ?= ${ROOT_DIR}/_artifacts - -ifeq ($(shell go env GOOS),darwin) # Use the darwin/amd64 binary until an arm64 version is available - KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path --arch amd64 $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION)) -else - KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use --use-env -p path $(KUBEBUILDER_ENVTEST_KUBERNETES_VERSION)) -endif - -.PHONY: test -test: $(SETUP_ENVTEST) ## Run unit and integration tests - KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test ./... $(TEST_ARGS) - -.PHONY: test-verbose -test-verbose: ## Run unit and integration tests with verbose flag - $(MAKE) test TEST_ARGS="$(TEST_ARGS) -v" - -.PHONY: test-junit -test-junit: $(SETUP_ENVTEST) $(GOTESTSUM) ## Run unit and integration tests and generate a junit report - set +o errexit; (KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -json ./... $(TEST_ARGS); echo $$? > $(ARTIFACTS)/junit.exitcode) | tee $(ARTIFACTS)/junit.stdout - $(GOTESTSUM) --junitfile $(ARTIFACTS)/junit.xml --raw-command cat $(ARTIFACTS)/junit.stdout - exit $$(cat $(ARTIFACTS)/junit.exitcode) - -.PHONY: test-cover -test-cover: ## Run unit and integration tests and generate a coverage report - $(MAKE) test TEST_ARGS="$(TEST_ARGS) -coverprofile=out/coverage.out" - go tool cover -func=out/coverage.out -o out/coverage.txt - go tool cover -html=out/coverage.out -o out/coverage.html - -.PHONY: test-e2e -test-e2e: ## Run e2e tests - $(MAKE) -C $(TEST_DIR)/e2e run - -.PHONY: test-e2e-k3s -test-e2e-k3s: ## Run e2e tests - $(MAKE) -C $(TEST_DIR)/e2e run-k3s - -## -------------------------------------- -## Release -## -------------------------------------- - -##@ release: - -## latest git tag for the commit, e.g., v0.3.10 -RELEASE_TAG ?= $(shell git describe --abbrev=0 2>/dev/null) -ifneq (,$(findstring -,$(RELEASE_TAG))) - PRE_RELEASE=true -endif -# the previous release tag, e.g., v0.3.9, excluding pre-release tags -PREVIOUS_TAG ?= $(shell git tag -l | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+$$" | sort -V | grep -B1 $(RELEASE_TAG) | head -n 1 2>/dev/null) -RELEASE_DIR := out - -$(RELEASE_DIR): - mkdir -p $(RELEASE_DIR)/ - -.PHONY: release -release: clean-release ## Build and push container images using the latest git tag for the commit - @if [ -z "${RELEASE_TAG}" ]; then echo "RELEASE_TAG is not set"; exit 1; fi - @if ! [ -z "$$(git status --porcelain)" ]; then echo "Your local git repository contains uncommitted changes, use git clean before proceeding."; exit 1; fi - git checkout "${RELEASE_TAG}" - ## Build binaries first. - GIT_VERSION=$(RELEASE_TAG) $(MAKE) release-binaries - # Set the manifest image to the production bucket. - $(MAKE) manifest-modification REGISTRY=$(PROD_REGISTRY) - ## Build the manifests - $(MAKE) release-manifests - ## Build the templates - $(MAKE) release-templates - ## Clean the git artifacts modified in the release process - $(MAKE) clean-release-git - -release-binaries: ## Build the binaries to publish with a release - RELEASE_BINARY=./cmd/kk GOOS=linux GOARCH=amd64 $(MAKE) release-binary - RELEASE_BINARY=./cmd/kk GOOS=linux GOARCH=amd64 $(MAKE) release-archive - RELEASE_BINARY=./cmd/kk GOOS=linux GOARCH=arm64 $(MAKE) release-binary - RELEASE_BINARY=./cmd/kk GOOS=linux GOARCH=arm64 $(MAKE) release-archive - RELEASE_BINARY=./cmd/kk GOOS=darwin GOARCH=amd64 $(MAKE) release-binary - RELEASE_BINARY=./cmd/kk GOOS=darwin GOARCH=amd64 $(MAKE) release-archive - RELEASE_BINARY=./cmd/kk GOOS=darwin GOARCH=arm64 $(MAKE) release-binary - RELEASE_BINARY=./cmd/kk GOOS=darwin GOARCH=arm64 $(MAKE) release-archive - -release-binary: $(RELEASE_DIR) - docker run \ - --rm \ - -e CGO_ENABLED=0 \ - -e GOOS=$(GOOS) \ - -e GOARCH=$(GOARCH) \ - -e GOPROXY=$(GOPROXY) \ - -v "$$(pwd):/workspace$(DOCKER_VOL_OPTS)" \ - -w /workspace \ - golang:$(GO_VERSION) \ - go build -a -trimpath -tags "$(BUILDTAGS)" -ldflags "$(LDFLAGS) -extldflags '-static'" \ - -o $(RELEASE_DIR)/$(notdir $(RELEASE_BINARY)) $(RELEASE_BINARY) - -release-archive: $(RELEASE_DIR) - tar -czf $(RELEASE_DIR)/kubekey-$(RELEASE_TAG)-$(GOOS)-$(GOARCH).tar.gz -C $(RELEASE_DIR)/ $(notdir $(RELEASE_BINARY)) - rm -rf $(RELEASE_DIR)/$(notdir $(RELEASE_BINARY)) - -.PHONY: manifest-modification -manifest-modification: # Set the manifest images to the staging/production bucket. - $(MAKE) set-manifest-image \ - MANIFEST_IMG=$(REGISTRY)/$(CAPKK_IMAGE_NAME) MANIFEST_TAG=$(RELEASE_TAG) \ - TARGET_RESOURCE="./config/default/manager_image_patch.yaml" - $(MAKE) set-manifest-image \ - MANIFEST_IMG=$(REGISTRY)/$(K3S_BOOTSTRAP_IMAGE_NAME) MANIFEST_TAG=$(RELEASE_TAG) \ - TARGET_RESOURCE="./bootstrap/k3s/config/default/manager_image_patch.yaml" - $(MAKE) set-manifest-image \ - MANIFEST_IMG=$(REGISTRY)/$(K3S_CONTROL_PLANE_IMAGE_NAME) MANIFEST_TAG=$(RELEASE_TAG) \ - TARGET_RESOURCE="./controlplane/k3s/config/default/manager_image_patch.yaml" - $(MAKE) set-manifest-pull-policy PULL_POLICY=IfNotPresent TARGET_RESOURCE="./config/default/manager_pull_policy.yaml" - $(MAKE) set-manifest-pull-policy PULL_POLICY=IfNotPresent TARGET_RESOURCE="./bootstrap/k3s/config/default/manager_pull_policy.yaml" - $(MAKE) set-manifest-pull-policy PULL_POLICY=IfNotPresent TARGET_RESOURCE="./controlplane/k3s/config/default/manager_pull_policy.yaml" - -.PHONY: release-manifests -release-manifests: $(RELEASE_DIR) $(KUSTOMIZE) ## Build the manifests to publish with a release - # Build capkk-components. - $(KUSTOMIZE) build config/default > $(RELEASE_DIR)/infrastructure-components.yaml - # Build bootstrap-components. - $(KUSTOMIZE) build bootstrap/k3s/config/default > $(RELEASE_DIR)/bootstrap-components.yaml - # Build control-plane-components. - $(KUSTOMIZE) build controlplane/k3s/config/default > $(RELEASE_DIR)/control-plane-components.yaml - - # Add metadata to the release artifacts - cp metadata.yaml $(RELEASE_DIR)/metadata.yaml - -.PHONY: release-templates -release-templates: $(RELEASE_DIR) ## Generate release templates - cp templates/cluster-template*.yaml $(RELEASE_DIR)/ - -.PHONY: release-prod -release-prod: ## Build and push container images to the prod - REGISTRY=$(PROD_REGISTRY) TAG=$(RELEASE_TAG) $(MAKE) docker-build-all docker-push-all - -## -------------------------------------- -## Docker -## -------------------------------------- - -.PHONY: docker-push-all -docker-push-all: $(addprefix docker-push-,$(ALL_ARCH)) ## Push the docker images to be included in the release for all architectures + related multiarch manifests - $(MAKE) docker-push-manifest-capkk - $(MAKE) docker-push-manifest-k3s-bootstrap - $(MAKE) docker-push-manifest-k3s-control-plane - -docker-push-%: - $(MAKE) ARCH=$* docker-push - -.PHONY: docker-push -docker-push: ## Push the docker images - docker push $(CAPKK_CONTROLLER_IMG)-$(ARCH):$(TAG) - docker push $(K3S_BOOTSTRAP_CONTROLLER_IMG)-$(ARCH):$(TAG) - docker push $(K3S_CONTROL_PLANE_CONTROLLER_IMG)-$(ARCH):$(TAG) - -.PHONY: docker-push-manifest-capkk -docker-push-manifest-capkk: ## Push the multiarch manifest for the capkk docker images - ## Minimum docker version 18.06.0 is required for creating and pushing manifest images. - docker manifest create --amend $(CAPKK_CONTROLLER_IMG):$(TAG) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(CAPKK_CONTROLLER_IMG)\-&:$(TAG)~g") - @for arch in $(ALL_ARCH); do docker manifest annotate --arch $${arch} ${CAPKK_CONTROLLER_IMG}:${TAG} ${CAPKK_CONTROLLER_IMG}-$${arch}:${TAG}; done - docker manifest push --purge $(CAPKK_CONTROLLER_IMG):$(TAG) - -.PHONY: docker-push-manifest-k3s-bootstrap -docker-push-manifest-k3s-bootstrap: ## Push the multiarch manifest for the k3s bootstrap docker images - ## Minimum docker version 18.06.0 is required for creating and pushing manifest images. - docker manifest create --amend $(K3S_BOOTSTRAP_CONTROLLER_IMG):$(TAG) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(K3S_BOOTSTRAP_CONTROLLER_IMG)\-&:$(TAG)~g") - @for arch in $(ALL_ARCH); do docker manifest annotate --arch $${arch} ${K3S_BOOTSTRAP_CONTROLLER_IMG}:${TAG} ${K3S_BOOTSTRAP_CONTROLLER_IMG}-$${arch}:${TAG}; done - docker manifest push --purge $(K3S_BOOTSTRAP_CONTROLLER_IMG):$(TAG) - -.PHONY: docker-push-manifest-k3s-control-plane -docker-push-manifest-k3s-control-plane: ## Push the multiarch manifest for the k3s control plane docker images - ## Minimum docker version 18.06.0 is required for creating and pushing manifest images. - docker manifest create --amend $(K3S_CONTROL_PLANE_CONTROLLER_IMG):$(TAG) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(K3S_CONTROL_PLANE_CONTROLLER_IMG)\-&:$(TAG)~g") - @for arch in $(ALL_ARCH); do docker manifest annotate --arch $${arch} ${K3S_CONTROL_PLANE_CONTROLLER_IMG}:${TAG} ${K3S_CONTROL_PLANE_CONTROLLER_IMG}-$${arch}:${TAG}; done - docker manifest push --purge $(K3S_CONTROL_PLANE_CONTROLLER_IMG):$(TAG) - -.PHONY: set-manifest-pull-policy -set-manifest-pull-policy: - $(info Updating kustomize pull policy file for manager resources) - sed -i'' -e 's@imagePullPolicy: .*@imagePullPolicy: '"$(PULL_POLICY)"'@' $(TARGET_RESOURCE) - -.PHONY: set-manifest-image -set-manifest-image: - $(info Updating kustomize image patch file for manager resource) - sed -i'' -e 's@image: .*@image: '"${MANIFEST_IMG}:$(MANIFEST_TAG)"'@' $(TARGET_RESOURCE) - -## -------------------------------------- -## Cleanup / Verification -## -------------------------------------- - -##@ clean: - -.PHONY: clean -clean: ## Remove all generated files - $(MAKE) clean-bin - -.PHONY: clean-bin -clean-bin: ## Remove all generated binaries - rm -rf $(BIN_DIR) - rm -rf $(TOOLS_BIN_DIR) - -.PHONY: clean-release -clean-release: ## Remove the release folder - rm -rf $(RELEASE_DIR) - -.PHONY: clean-release-git -clean-release-git: ## Restores the git files usually modified during a release - git restore ./*manager_image_patch.yaml ./*manager_pull_policy.yaml - -.PHONY: clean-generated-yaml -clean-generated-yaml: ## Remove files generated by conversion-gen from the mentioned dirs. Example SRC_DIRS="./api/v1beta1" - (IFS=','; for i in $(SRC_DIRS); do find $$i -type f -name '*.yaml' -exec rm -f {} \;; done) - -.PHONY: clean-generated-deepcopy -clean-generated-deepcopy: ## Remove files generated by conversion-gen from the mentioned dirs. Example SRC_DIRS="./api/v1beta1" - (IFS=','; for i in $(SRC_DIRS); do find $$i -type f -name 'zz_generated.deepcopy*' -exec rm -f {} \;; done) - -## -------------------------------------- -## Hack / Tools -## -------------------------------------- - -##@ hack/tools: - -.PHONY: $(CONTROLLER_GEN_BIN) -$(CONTROLLER_GEN_BIN): $(CONTROLLER_GEN) ## Build a local copy of controller-gen. - -.PHONY: $(GOTESTSUM_BIN) -$(GOTESTSUM_BIN): $(GOTESTSUM) ## Build a local copy of gotestsum. - -.PHONY: $(KUSTOMIZE_BIN) -$(KUSTOMIZE_BIN): $(KUSTOMIZE) ## Build a local copy of kustomize. - -.PHONY: $(SETUP_ENVTEST_BIN) -$(SETUP_ENVTEST_BIN): $(SETUP_ENVTEST) ## Build a local copy of setup-envtest. - -.PHONY: $(GOLANGCI_LINT_BIN) -$(GOLANGCI_LINT_BIN): $(GOLANGCI_LINT) ## Build a local copy of golangci-lint - -$(CONTROLLER_GEN): # Build controller-gen from tools folder. - GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(CONTROLLER_GEN_PKG) $(CONTROLLER_GEN_BIN) $(CONTROLLER_GEN_VER) - -$(GOTESTSUM): # Build gotestsum from tools folder. - GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(GOTESTSUM_PKG) $(GOTESTSUM_BIN) $(GOTESTSUM_VER) - -$(KUSTOMIZE): # Build kustomize from tools folder. - CGO_ENABLED=0 GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(KUSTOMIZE_PKG) $(KUSTOMIZE_BIN) $(KUSTOMIZE_VER) - -$(SETUP_ENVTEST): # Build setup-envtest from tools folder. - GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(SETUP_ENVTEST_PKG) $(SETUP_ENVTEST_BIN) $(SETUP_ENVTEST_VER) - -$(GOLANGCI_LINT): .github/workflows/golangci-lint.yml # Download golangci-lint using hack script into tools folder. - hack/ensure-golangci-lint.sh \ - -b $(TOOLS_BIN_DIR) \ - $(shell cat .github/workflows/golangci-lint.yml | grep [[:space:]]version | sed 's/.*version: //') - -# build the artifact of repository iso -ISO_ARCH ?= amd64 -ISO_OUTPUT_DIR ?= ./output -ISO_BUILD_WORKDIR := hack/gen-repository-iso -ISO_OS_NAMES := centos7 debian9 debian10 ubuntu1604 ubuntu1804 ubuntu2004 ubuntu2204 -ISO_BUILD_NAMES := $(addprefix build-iso-,$(ISO_OS_NAMES)) -build-iso-all: $(ISO_BUILD_NAMES) -.PHONY: $(ISO_BUILD_NAMES) -$(ISO_BUILD_NAMES): - @export DOCKER_BUILDKIT=1 - docker build \ - --platform linux/$(ISO_ARCH) \ - --build-arg TARGETARCH=$(ISO_ARCH) \ - -o type=local,dest=$(ISO_OUTPUT_DIR) \ - -f $(ISO_BUILD_WORKDIR)/dockerfile.$(subst build-iso-,,$@) \ - $(ISO_BUILD_WORKDIR) - -go-releaser-test: - goreleaser release --rm-dist --skip-publish --snapshot diff --git a/OWNERS b/OWNERS deleted file mode 100644 index 6492db82..00000000 --- a/OWNERS +++ /dev/null @@ -1,20 +0,0 @@ -approvers: - - pixiake - - 24sama - - rayzhou2017 - - littleBlackHouse - -reviewers: - - pixiake - - Forest-L - - rayzhou2017 - - zryfish - - shaowenchen - - benjaminhuo - - calvinyv - - FeynmanZhou - - huanggze - - wansir - - LinuxSuRen - - 24sama - - littleBlackHouse diff --git a/PROJECT b/PROJECT deleted file mode 100644 index d710397d..00000000 --- a/PROJECT +++ /dev/null @@ -1,73 +0,0 @@ -domain: cluster.x-k8s.io -layout: -- go.kubebuilder.io/v3 -plugins: - manifests.sdk.operatorframework.io/v2: {} - scorecard.sdk.operatorframework.io/v2: {} -projectName: cluster-api-provider-kubekey -repo: github.com/kubesphere/kubekey -resources: -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: cluster.x-k8s.io - group: infrastructure - kind: KKCluster - path: github.com/kubesphere/kubekey/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -- api: - crdVersion: v1 - namespaced: true - domain: cluster.x-k8s.io - group: infrastructure - kind: KKClusterTemplate - path: github.com/kubesphere/kubekey/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: cluster.x-k8s.io - group: infrastructure - kind: KKMachine - path: github.com/kubesphere/kubekey/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -- api: - crdVersion: v1 - namespaced: true - domain: cluster.x-k8s.io - group: infrastructure - kind: KKMachineTemplate - path: github.com/kubesphere/kubekey/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: cluster.x-k8s.io - group: infrastructure - kind: KKInstance - path: github.com/kubesphere/kubekey/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -version: "3" diff --git a/README.md b/README.md deleted file mode 100644 index 3f058c8d..00000000 --- a/README.md +++ /dev/null @@ -1,408 +0,0 @@ -
- -[![CI](https://github.com/kubesphere/kubekey/workflows/CI/badge.svg?branch=master&event=push)](https://github.com/kubesphere/kubekey/actions?query=event%3Apush+branch%3Amaster+workflow%3ACI+) - -> English | [中文](README_zh-CN.md) - -### 👋 Welcome to KubeKey! - -KubeKey is an open-source lightweight tool for deploying Kubernetes clusters. It provides a flexible, rapid, and convenient way to install Kubernetes/K3s only, both Kubernetes/K3s and KubeSphere, and related cloud-native add-ons. It is also an efficient tool to scale and upgrade your cluster. - -In addition, KubeKey also supports customized Air-Gap package, which is convenient for users to quickly deploy clusters in offline environments. - -> KubeKey has passed [CNCF kubernetes conformance verification](https://www.cncf.io/certification/software-conformance/). - -Use KubeKey in the following three scenarios. - -* Install Kubernetes/K3s only -* Install Kubernetes/K3s and KubeSphere together in one command -* Install Kubernetes/K3s first, then deploy KubeSphere on it using [ks-installer](https://github.com/kubesphere/ks-installer) - -> **Important:** If you have existing Kubernetes clusters, please refer to [ks-installer (Install KubeSphere on existing Kubernetes cluster)](https://github.com/kubesphere/ks-installer). - -## Supported Environment - -### Linux Distributions - -* **Ubuntu** *16.04, 18.04, 20.04, 22.04* -* **Debian** *Bullseye, Buster, Stretch* -* **CentOS/RHEL** *7* -* **AlmaLinux** *9.0* -* **SUSE Linux Enterprise Server** *15* - -> Recommended Linux Kernel Version: `4.15 or later` -> You can run the `uname -srm` command to check the Linux Kernel Version. - -### Kubernetes Versions - -* **v1.19**:   *v1.19.15* -* **v1.20**:   *v1.20.10* -* **v1.21**:   *v1.21.14* -* **v1.22**:   *v1.22.15* -* **v1.23**:   *v1.23.10* (default) -* **v1.24**:   *v1.24.7* -* **v1.25**:   *v1.25.3* - -> Looking for more supported versions: \ -> [Kubernetes Versions](./docs/kubernetes-versions.md) \ -> [K3s Versions](./docs/k3s-versions.md) - -### Container Manager - -* **Docker** / **containerd** / **CRI-O** / **iSula** - -> `Kata Containers` can be set to automatically install and configure runtime class for it when the container manager is containerd or CRI-O. - -### Network Plugins - -* **Calico** / **Flannel** / **Cilium** / **Kube-OVN** / **Multus-CNI** - -> Kubekey also supports users to set the network plugin to `none` if there is a requirement for custom network plugin. - -## Requirements and Recommendations - -* Minimum resource requirements (For Minimal Installation of KubeSphere only)： - * 2 vCPUs - * 4 GB RAM - * 20 GB Storage - -> /var/lib/docker is mainly used to store the container data, and will gradually increase in size during use and operation. In the case of a production environment, it is recommended that /var/lib/docker mounts a drive separately. - -* OS requirements: - * `SSH` can access to all nodes. - * Time synchronization for all nodes. - * `sudo`/`curl`/`openssl` should be used in all nodes. - * `docker` can be installed by yourself or by KubeKey. - * `Red Hat` includes `SELinux` in its `Linux release`. It is recommended to close SELinux or [switch the mode of SELinux](./docs/turn-off-SELinux.md) to `Permissive` - -> * It's recommended that Your OS is clean (without any other software installed), otherwise there may be conflicts. -> * A container image mirror (accelerator) is recommended to be prepared if you have trouble downloading images from dockerhub.io. [Configure registry-mirrors for the Docker daemon](https://docs.docker.com/registry/recipes/mirror/#configure-the-docker-daemon). -> * KubeKey will install [OpenEBS](https://openebs.io/) to provision LocalPV for development and testing environment by default, this is convenient for new users. For production, please use NFS / Ceph / GlusterFS or commercial products as persistent storage, and install the [relevant client](docs/storage-client.md) in all nodes. -> * If you encounter `Permission denied` when copying, it is recommended to check [SELinux and turn off it](./docs/turn-off-SELinux.md) first - -* Dependency requirements: - -KubeKey can install Kubernetes and KubeSphere together. Some dependencies need to be installed before installing kubernetes after version 1.18. You can refer to the list below to check and install the relevant dependencies on your node in advance. - -| | Kubernetes Version ≥ 1.18 | -| ------------- | -------------------------- | -| `socat` | Required | -| `conntrack` | Required | -| `ebtables` | Optional but recommended | -| `ipset` | Optional but recommended | -| `ipvsadm` | Optional but recommended | - -* Networking and DNS requirements: - * Make sure the DNS address in `/etc/resolv.conf` is available. Otherwise, it may cause some issues of DNS in cluster. - * If your network configuration uses Firewall or Security Group，you must ensure infrastructure components can communicate with each other through specific ports. It's recommended that you turn off the firewall or follow the link configuriation: [NetworkAccess](docs/network-access.md). - -## Usage - -### Get the KubeKey Executable File - -* The fastest way to get KubeKey is to use the script: - - ``` - curl -sfL https://get-kk.kubesphere.io | sh - - ``` -* Binary downloads of the KubeKey also can be found on the [Releases page](https://github.com/kubesphere/kubekey/releases). - Unpack the binary and you are good to go! -* Build Binary from Source Code - - ```shell - git clone https://github.com/kubesphere/kubekey.git - cd kubekey - make kk - ``` - -### Create a Cluster - -#### Quick Start - -Quick Start is for `all-in-one` installation which is a good start to get familiar with Kubernetes and KubeSphere. - -> Note: Since Kubernetes temporarily does not support uppercase NodeName, contains uppercase letters in the hostname will lead to subsequent installation error - -##### Command - -> If you have problem to access `https://storage.googleapis.com`, execute first `export KKZONE=cn`. - -```shell -./kk create cluster [--with-kubernetes version] [--with-kubesphere version] -``` - -##### Examples - -* Create a pure Kubernetes cluster with default version (Kubernetes v1.23.10). - - ```shell - ./kk create cluster - ``` -* Create a Kubernetes cluster with a specified version. - - ```shell - ./kk create cluster --with-kubernetes v1.24.1 --container-manager containerd - ``` -* Create a Kubernetes cluster with KubeSphere installed. - - ```shell - ./kk create cluster --with-kubesphere v3.2.1 - ``` - -#### Advanced - -You have more control to customize parameters or create a multi-node cluster using the advanced installation. Specifically, create a cluster by specifying a configuration file. - -> If you have problem to access `https://storage.googleapis.com`, execute first `export KKZONE=cn`. - -1. First, create an example configuration file - - ```shell - ./kk create config [--with-kubernetes version] [--with-kubesphere version] [(-f | --filename) path] - ``` - - **examples:** - - * create an example config file with default configurations. You also can specify the file that could be a different filename, or in different folder. - - ```shell - ./kk create config [-f ~/myfolder/abc.yaml] - ``` - - * with KubeSphere - - ```shell - ./kk create config --with-kubesphere v3.2.1 - ``` -2. Modify the file config-sample.yaml according to your environment - -> Note: Since Kubernetes temporarily does not support uppercase NodeName, contains uppercase letters in workerNode`s name will lead to subsequent installation error -> -> A persistent storage is required in the cluster, when kubesphere will be installed. The local volume is used default. If you want to use other persistent storage, please refer to [addons](./docs/addons.md). - -3. Create a cluster using the configuration file - - ```shell - ./kk create cluster -f config-sample.yaml - ``` - -### Enable Multi-cluster Management - -By default, KubeKey will only install a **solo** cluster without Kubernetes federation. If you want to set up a multi-cluster control plane to centrally manage multiple clusters using KubeSphere, you need to set the `ClusterRole` in [config-example.yaml](docs/config-example.md). For multi-cluster user guide, please refer to [How to Enable the Multi-cluster Feature](https://github.com/kubesphere/community/tree/master/sig-multicluster/how-to-setup-multicluster-on-kubesphere). - -### Enable Pluggable Components - -KubeSphere has decoupled some core feature components since v2.1.0. These components are designed to be pluggable which means you can enable them either before or after installation. By default, KubeSphere will be started with a minimal installation if you do not enable them. - -You can enable any of them according to your demands. It is highly recommended that you install these pluggable components to discover the full-stack features and capabilities provided by KubeSphere. Please ensure your machines have sufficient CPU and memory before enabling them. See [Enable Pluggable Components](https://github.com/kubesphere/ks-installer#enable-pluggable-components) for the details. - -### Add Nodes - -Add new node's information to the cluster config file, then apply the changes. - -```shell -./kk add nodes -f config-sample.yaml -``` - -### Delete Nodes - -You can delete the node by the following command，the nodeName that needs to be removed. - -```shell -./kk delete node -f config-sample.yaml -``` - -### Delete Cluster - -You can delete the cluster by the following command: - -* If you started with the quick start (all-in-one): - -```shell -./kk delete cluster -``` - -* If you started with the advanced (created with a configuration file): - -```shell -./kk delete cluster [-f config-sample.yaml] -``` - -### Upgrade Cluster - -#### Allinone - -Upgrading cluster with a specified version. - -```shell -./kk upgrade [--with-kubernetes version] [--with-kubesphere version] -``` - -* Support upgrading Kubernetes only. -* Support upgrading KubeSphere only. -* Support upgrading Kubernetes and KubeSphere. - -#### Multi-nodes - -Upgrading cluster with a specified configuration file. - -```shell -./kk upgrade [--with-kubernetes version] [--with-kubesphere version] [(-f | --filename) path] -``` - -* If `--with-kubernetes` or `--with-kubesphere` is specified, the configuration file will be also updated. -* Use `-f` to specify the configuration file which was generated for cluster creation. - -> Note: Upgrading multi-nodes cluster need a specified configuration file. If the cluster was installed without kubekey or the configuration file for installation was not found, the configuration file needs to be created by yourself or following command. - -Getting cluster info and generating kubekey's configuration file (optional). - -```shell -./kk create config [--from-cluster] [(-f | --filename) path] [--kubeconfig path] -``` - -* `--from-cluster` means fetching cluster's information from an existing cluster. -* `-f` refers to the path where the configuration file is generated. -* `--kubeconfig` refers to the path where the kubeconfig. -* After generating the configuration file, some parameters need to be filled in, such as the ssh information of the nodes. - -## Documents - -* [Features List](docs/features.md) -* [Commands](docs/commands/kk.md) -* [Configuration example](docs/config-example.md) -* [Air-Gapped Installation](docs/manifest_and_artifact.md) -* [Highly Available clusters](docs/ha-mode.md) -* [Addons](docs/addons.md) -* [Network access](docs/network-access.md) -* [Storage clients](docs/storage-client.md) -* [kubectl auto-completion](docs/kubectl-autocompletion.md) -* [kubekey auto-completion](docs/kubekey-autocompletion.md) -* [Roadmap](docs/roadmap.md) -* [Check-Renew-Certificate](docs/check-renew-certificate.md) -* [Developer-Guide](docs/developer-guide.md) - -## Contributors ✨ - -Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
pixiake
pixiake
💻 📖		Forest
Forest
💻 📖		rayzhou2017
rayzhou2017
💻 📖		shaowenchen
shaowenchen
💻 📖		Zhao Xiaojie
Zhao Xiaojie
💻 📖		Zack Zhang
Zack Zhang
💻		Akhil Mohan
Akhil Mohan
💻
pengfei
pengfei
📖		min zhang
min zhang
💻 📖		zgldh
zgldh
💻		xrjk
xrjk
💻		yonghongshi
yonghongshi
💻		Honglei
Honglei
📖		liucy1983
liucy1983
💻
Lien
Lien
📖		Tony Wang
Tony Wang
📖		Hongliang Wang
Hongliang Wang
💻		dawn
dawn
💻		Duan Jiong
Duan Jiong
💻		calvinyv
calvinyv
📖		Benjamin Huo
Benjamin Huo
📖
Sherlock113
Sherlock113
📖		fu_changjie
fu_changjie
📖		yuswift
yuswift
💻		ruiyaoOps
ruiyaoOps
📖		LXM
LXM
📖		sbhnet
sbhnet
💻		misteruly
misteruly
💻
John Niang
John Niang
📖		Michael Li
Michael Li
💻		独孤昊天
独孤昊天
💻		Liu Shaohui
Liu Shaohui
💻		Leo Li
Leo Li
💻		Roland
Roland
💻		Vinson Zou
Vinson Zou
📖
tag_gee_y
tag_gee_y
💻		codebee
codebee
💻		Daniel Owen van Dommelen
Daniel Owen van Dommelen
🤔		Naidile P N
Naidile P N
💻		Haiker Sun
Haiker Sun
💻		Jing Yu
Jing Yu
💻		Chauncey
Chauncey
💻
Tan Guofu
Tan Guofu
💻		lvillis
lvillis
📖		Vincent He
Vincent He
💻		laminar
laminar
💻		tongjin
tongjin
💻		Reimu
Reimu
💻		Ikko Ashimine
Ikko Ashimine
📖
Ben Ye
Ben Ye
💻		yinheli
yinheli
💻		hellocn9
hellocn9
💻		Brandan Schmitz
Brandan Schmitz
💻		yjqg6666
yjqg6666
📖 💻		失眠是真滴难受
失眠是真滴难受
💻		mango
mango
👀
wenwutang
wenwutang
💻		Shiny Hou
Shiny Hou
💻		zhouqiu0103
zhouqiu0103
💻		77yu77
77yu77
💻		hzhhong
hzhhong
💻		zhang-wei
zhang-wei
💻		Deshi Xiao
Deshi Xiao
💻 📖
besscroft
besscroft
📖		张志强
张志强
💻		lwabish
lwabish
💻 📖		qyz87
qyz87
💻		ZhengJin Fang
ZhengJin Fang
💻		Eric_Lian
Eric_Lian
💻		nicognaw
nicognaw
💻
吕德庆
吕德庆
💻		littleplus
littleplus
💻		Konstantin
Konstantin
🤔		kiragoo
kiragoo
💻		jojotong
jojotong
💻		littleBlackHouse
littleBlackHouse
💻 📖		guangwu
guangwu
💻 📖
wongearl
wongearl
💻		wenwenxiong
wenwenxiong
💻		柏喵Sakura
柏喵Sakura
💻		cui fliter
cui fliter
📖		刘旭
刘旭
💻
- - - - - - -This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! diff --git a/README_zh-CN.md b/README_zh-CN.md deleted file mode 100644 index 753920d7..00000000 --- a/README_zh-CN.md +++ /dev/null @@ -1,423 +0,0 @@ -# KubeKey - -[![CI](https://github.com/kubesphere/kubekey/workflows/CI/badge.svg?branch=master&event=push)](https://github.com/kubesphere/kubekey/actions?query=event%3Apush+branch%3Amaster+workflow%3ACI+) - -> [English](README.md) | 中文 - -KubeKey是一个开源的轻量级工具，用于部署Kubernetes集群。它提供了一种灵活、快速、方便的方式来安装Kubernetes/K3s、Kubernetes/K3s和KubeSphere，以及相关的云原生附加组件。它也是扩展和升级集群的有效工具。 - -此外，KubeKey还支持定制离线包（artifact），方便用户在离线环境下快速部署集群。 - -> KubeKey 通过了 [CNCF kubernetes 一致性认证](https://www.cncf.io/certification/software-conformance/)。 - -有三种情况可以使用 KubeKey。 - -* 仅安装 Kubernetes -* 用一个命令中安装 Kubernetes 和 KubeSphere -* 首先安装 Kubernetes，然后使用 [ks-installer](https://github.com/kubesphere/ks-installer) 在其上部署 KubeSphere - -> 重要提示：Kubekey 将会帮您安装 Kubernetes，若已有 Kubernetes 集群请参考 [在 Kubernetes 之上安装 KubeSphere](https://github.com/kubesphere/ks-installer/)。 - -## 优势 - -* 基于 Ansible 的安装程序具有大量软件依赖性，例如 Python。KubeKey 是使用 Go 语言开发的，可以消除在各种环境中出现的问题，从而提高安装成功率。 -* KubeKey 使用 Kubeadm 在节点上尽可能多地并行安装 K8s 集群，以降低安装复杂性并提高效率。与较早的安装程序相比，它将大大节省安装时间。 -* KubeKey 支持将集群从 all-in-one 扩展到多节点集群甚至 HA 集群。 -* KubeKey 旨在将集群当作一个对象操作，即 CaaO。 - -## 支持的环境 - -### Linux 发行版 - -* **Ubuntu** *16.04, 18.04, 20.04, 22.04* -* **Debian** *Bullseye, Buster, Stretch* -* **CentOS/RHEL** *7* -* **AlmaLinux** *9.0* -* **SUSE Linux Enterprise Server** *15* - -> 建议使用 Linux Kernel 版本: `4.15 or later` -> 可以通过命令 `uname -srm` 查看 Linux Kernel 版本。 - -### Kubernetes 版本 - -* **v1.19**:   *v1.19.15* -* **v1.20**:   *v1.20.10* -* **v1.21**:   *v1.21.14* -* **v1.22**:   *v1.22.15* -* **v1.23**:   *v1.23.10* (default) -* **v1.24**:   *v1.24.7* -* **v1.25**:   *v1.25.3* - -> 查看更多支持的版本： \ -> [Kubernetes 版本](./docs/kubernetes-versions.md) \ -> [K3s 版本](./docs/k3s-versions.md) - -## 要求和建议 - -* 最低资源要求（仅对于最小安装 KubeSphere）： - * 2 核虚拟 CPU - * 4 GB 内存 - * 20 GB 储存空间 - -> /var/lib/docker 主要用于存储容器数据，在使用和操作过程中会逐渐增大。对于生产环境，建议 /var/lib/docker 单独挂盘。 - -* 操作系统要求： - * `SSH` 可以访问所有节点。 - * 所有节点的时间同步。 - * `sudo`/`curl`/`openssl` 应在所有节点使用。 - * `docker` 可以自己安装，也可以通过 KubeKey 安装。 - * `Red Hat` 在其 `Linux` 发行版本中包括了 `SELinux`，建议[关闭SELinux](./docs/turn-off-SELinux_zh-CN.md)或者将[SELinux的模式切换](./docs/turn-off-SELinux_zh-CN.md)为Permissive[宽容]工作模式 - -> * 建议您的操作系统环境足够干净 (不安装任何其他软件)，否则可能会发生冲突。 -> * 如果在从 dockerhub.io 下载镜像时遇到问题，建议准备一个容器镜像仓库 (加速器)。[为 Docker 守护程序配置镜像加速](https://docs.docker.com/registry/recipes/mirror/#configure-the-docker-daemon)。 -> * 默认情况下，KubeKey 将安装 [OpenEBS](https://openebs.io/) 来为开发和测试环境配置 LocalPV，这对新用户来说非常方便。对于生产，请使用 NFS/Ceph/GlusterFS 或商业化存储作为持久化存储，并在所有节点中安装[相关的客户端](./docs/storage-client.md) 。 -> * 如果遇到拷贝时报权限问题Permission denied,建议优先考虑查看[SELinux的原因](./docs/turn-off-SELinux_zh-CN.md)。 - -* 依赖要求: - -KubeKey 可以同时安装 Kubernetes 和 KubeSphere。在版本1.18之后，安装kubernetes前需要安装一些依赖。你可以参考下面的列表，提前在你的节点上检查并安装相关依赖。 - -| | Kubernetes 版本 ≥ 1.18 | -| ------------- | ----------------------- | -| `socat` | 必须安装 | -| `conntrack` | 必须安装 | -| `ebtables` | 可选，但推荐安装 | -| `ipset` | 可选，但推荐安装 | -| `ipvsadm` | 可选，但推荐安装 | - -* 网络和 DNS 要求： - * 确保 `/etc/resolv.conf` 中的 DNS 地址可用。否则，可能会导致集群中出现某些 DNS 问题。 - * 如果您的网络配置使用防火墙或安全组，则必须确保基础结构组件可以通过特定端口相互通信。建议您关闭防火墙或遵循链接配置：[网络访问](docs/network-access.md)。 - -## 用法 - -### 获取安装程序可执行文件 - -* 使用脚本获取 KubeKey - > 如果无法访问 https://github.com, 请先执行 export KKZONE=cn. - ``` - curl -sfL https://get-kk.kubesphere.io | sh - - ``` - -* 下载KubeKey可执行文件 [Releases page](https://github.com/kubesphere/kubekey/releases) - - 下载解压后可直接使用。 - -* 从源代码生成二进制文件 - - ```shell - git clone https://github.com/kubesphere/kubekey.git - cd kubekey - make kk - ``` - -### 创建集群 - -#### 快速开始 - -快速入门使用 `all-in-one` 安装，这是熟悉 KubeSphere 的良好开始。 - -> 注意： 由于 Kubernetes 暂不支持大写 NodeName， hostname 中包含大写字母将导致后续安装过程无法正常结束 - -##### 命令 - -> 如果无法访问 `https://storage.googleapis.com`, 请先执行 `export KKZONE=cn`. - -```shell -./kk create cluster [--with-kubernetes version] [--with-kubesphere version] -``` - -##### 例子 - -* 使用默认版本创建一个纯 Kubernetes 集群 - - ```shell - ./kk create cluster - ``` -* 创建指定一个（[支持的版本](#KubernetesVersions)）的 Kubernetes 集群 - - ```shell - ./kk create cluster --with-kubernetes v1.19.8 - ``` -* 创建一个部署了 KubeSphere 的 Kubernetes 集群 （例如 `--with-kubesphere v3.1.0`） - - ```shell - ./kk create cluster --with-kubesphere [version] - ``` -* 创建一个指定的 container runtime 的 Kubernetes 集群（docker, crio, containerd and isula） - - ```shell - ./kk create cluster --container-manager containerd - ``` - -#### 高级用法 - -您可以使用高级安装来控制自定义参数或创建多节点集群。具体来说，通过指定配置文件来创建集群。 - -> 如果无法访问 `https://storage.googleapis.com`, 请先执行 `export KKZONE=cn`. - -1. 首先，创建一个示例配置文件 - - ```shell - ./kk create config [--with-kubernetes version] [--with-kubesphere version] [(-f | --filename) path] - ``` - - **例子：** - - * 使用默认配置创建一个示例配置文件。您也可以指定文件名称或文件所在的文件夹。 - - ```shell - ./kk create config [-f ~/myfolder/config-sample.yaml] - ``` - * 同时安装 KubeSphere - - ```shell - ./kk create config --with-kubesphere - ``` -2. 根据您的环境修改配置文件 config-sample.yaml - -> 注意： 由于 Kubernetes 暂不支持大写 NodeName， worker 节点名中包含大写字母将导致后续安装过程无法正常结束 -> -> 当指定安装KubeSphere时，要求集群中有可用的持久化存储。默认使用localVolume，如果需要使用其他持久化存储，请参阅 [addons](./docs/addons.md) 配置。 - -3. 使用配置文件创建集群。 - - ```shell - ./kk create cluster -f ~/myfolder/config-sample.yaml - ``` - -### 启用多集群管理 - -默认情况下，Kubekey 将仅安装一个 Solo 模式的单集群，即未开启 Kubernetes 多集群联邦。如果您希望将 KubeSphere 作为一个支持多集群集中管理的中央面板，您需要在 [config-example.yaml](docs/config-example.md) 中设置 `ClusterRole`。关于多集群的使用文档，请参考 [如何启用多集群](https://github.com/kubesphere/community/blob/master/sig-multicluster/how-to-setup-multicluster-on-kubesphere/README_zh.md)。 - -### 开启可插拔功能组件 - -KubeSphere 从 2.1.0 版本开始对 Installer 的各功能组件进行了解耦，快速安装将默认仅开启最小化安装（Minimal Installation），Installer 支持在安装前或安装后自定义可插拔的功能组件的安装。使最小化安装更快速轻量且资源占用更少，也方便不同用户按需选择安装不同的功能组件。 - -KubeSphere 有多个可插拔功能组件，功能组件的介绍可参考 [配置示例](docs/config-example.md)。您可以根据需求，选择开启安装 KubeSphere 的可插拔功能组件。我们非常建议您开启这些功能组件来体验 KubeSphere 完整的功能以及端到端的解决方案。请在安装前确保您的机器有足够的 CPU 与内存资源。开启可插拔功能组件可参考 [开启可选功能组件](https://github.com/kubesphere/ks-installer/blob/master/README_zh.md#%E5%AE%89%E8%A3%85%E5%8A%9F%E8%83%BD%E7%BB%84%E4%BB%B6)。 - -### 添加节点 - -将新节点的信息添加到集群配置文件，然后应用更改。 - -```shell -./kk add nodes -f config-sample.yaml -``` - -### 删除节点 - -通过以下命令删除节点，nodename指需要删除的节点名。 - -```shell -./kk delete node -f config-sample.yaml -``` - -### 删除集群 - -您可以通过以下命令删除集群： - -* 如果您以快速入门（all-in-one）开始： - -```shell -./kk delete cluster -``` - -* 如果从高级安装开始（使用配置文件创建的集群）： - -```shell -./kk delete cluster [-f config-sample.yaml] -``` - -### 集群升级 - -#### 单节点集群 - -升级集群到指定版本。 - -```shell -./kk upgrade [--with-kubernetes version] [--with-kubesphere version] -``` - -* `--with-kubernetes` 指定kubernetes目标版本。 -* `--with-kubesphere` 指定kubesphere目标版本。 - -#### 多节点集群 - -通过指定配置文件对集群进行升级。 - -```shell -./kk upgrade [--with-kubernetes version] [--with-kubesphere version] [(-f | --filename) path] -``` - -* `--with-kubernetes` 指定kubernetes目标版本。 -* `--with-kubesphere` 指定kubesphere目标版本。 -* `-f` 指定集群安装时创建的配置文件。 - -> 注意: 升级多节点集群需要指定配置文件. 如果集群非kubekey创建，或者创建集群时生成的配置文件丢失，需要重新生成配置文件，或使用以下方法生成。 - -Getting cluster info and generating kubekey's configuration file (optional). - -```shell -./kk create config [--from-cluster] [(-f | --filename) path] [--kubeconfig path] -``` - -* `--from-cluster` 根据已存在集群信息生成配置文件. -* `-f` 指定生成配置文件路径. -* `--kubeconfig` 指定集群kubeconfig文件. -* 由于无法全面获取集群配置，生成配置文件后，请根据集群实际信息补全配置文件。 - -### 启用 kubectl 自动补全 - -KubeKey 不会启用 kubectl 自动补全功能。请参阅下面的指南并将其打开： - -**先决条件**：确保已安装 `bash-autocompletion` 并可以正常工作。 - -```shell -# 安装 bash-completion -apt-get install bash-completion - -# 将 completion 脚本添加到你的 ~/.bashrc 文件 -echo 'source <(kubectl completion bash)' >>~/.bashrc - -# 将 completion 脚本添加到 /etc/bash_completion.d 目录 -kubectl completion bash >/etc/bash_completion.d/kubectl -``` - -更详细的参考可以在[这里](https://kubernetes.io/docs/tasks/tools/install-kubectl/#enabling-shell-autocompletion)找到。 - -## 相关文档 - -* [特性列表](docs/features.md) -* [命令手册](docs/commands/kk.md) -* [配置示例](docs/config-example.md) -* [离线安装](docs/zh/manifest_and_artifact.md) -* [高可用集群](docs/ha-mode.md) -* [自定义插件安装](docs/addons.md) -* [网络访问](docs/network-access.md) -* [存储客户端](docs/storage-client.md) -* [路线图](docs/roadmap.md) -* [查看或更新证书](docs/check-renew-certificate.md) -* [开发指南](docs/developer-guide.md) - -## 贡献者 ✨ - -欢迎任何形式的贡献! 感谢这些优秀的贡献者，是他们让我们的项目快速成长。 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
pixiake
pixiake
💻 📖		Forest
Forest
💻 📖		rayzhou2017
rayzhou2017
💻 📖		shaowenchen
shaowenchen
💻 📖		Zhao Xiaojie
Zhao Xiaojie
💻 📖		Zack Zhang
Zack Zhang
💻		Akhil Mohan
Akhil Mohan
💻
pengfei
pengfei
📖		min zhang
min zhang
💻 📖		zgldh
zgldh
💻		xrjk
xrjk
💻		yonghongshi
yonghongshi
💻		Honglei
Honglei
📖		liucy1983
liucy1983
💻
Lien
Lien
📖		Tony Wang
Tony Wang
📖		Hongliang Wang
Hongliang Wang
💻		dawn
dawn
💻		Duan Jiong
Duan Jiong
💻		calvinyv
calvinyv
📖		Benjamin Huo
Benjamin Huo
📖
Sherlock113
Sherlock113
📖		fu_changjie
fu_changjie
📖		yuswift
yuswift
💻		ruiyaoOps
ruiyaoOps
📖		LXM
LXM
📖		sbhnet
sbhnet
💻		misteruly
misteruly
💻
John Niang
John Niang
📖		Michael Li
Michael Li
💻		独孤昊天
独孤昊天
💻		Liu Shaohui
Liu Shaohui
💻		Leo Li
Leo Li
💻		Roland
Roland
💻		Vinson Zou
Vinson Zou
📖
tag_gee_y
tag_gee_y
💻		codebee
codebee
💻		Daniel Owen van Dommelen
Daniel Owen van Dommelen
🤔		Naidile P N
Naidile P N
💻		Haiker Sun
Haiker Sun
💻		Jing Yu
Jing Yu
💻		Chauncey
Chauncey
💻
Tan Guofu
Tan Guofu
💻		lvillis
lvillis
📖		Vincent He
Vincent He
💻		laminar
laminar
💻		tongjin
tongjin
💻		Reimu
Reimu
💻		Ikko Ashimine
Ikko Ashimine
📖
Ben Ye
Ben Ye
💻		yinheli
yinheli
💻		hellocn9
hellocn9
💻		Brandan Schmitz
Brandan Schmitz
💻		yjqg6666
yjqg6666
📖 💻		失眠是真滴难受
失眠是真滴难受
💻		mango
mango
👀
wenwutang
wenwutang
💻		Shiny Hou
Shiny Hou
💻		zhouqiu0103
zhouqiu0103
💻		77yu77
77yu77
💻		hzhhong
hzhhong
💻		zhang-wei
zhang-wei
💻		Deshi Xiao
Deshi Xiao
💻 📖
besscroft
besscroft
📖		张志强
张志强
💻		lwabish
lwabish
💻 📖		qyz87
qyz87
💻		ZhengJin Fang
ZhengJin Fang
💻		Eric_Lian
Eric_Lian
💻		nicognaw
nicognaw
💻
吕德庆
吕德庆
💻		littleplus
littleplus
💻		Konstantin
Konstantin
🤔		kiragoo
kiragoo
💻		jojotong
jojotong
💻		littleBlackHouse
littleBlackHouse
💻 📖		guangwu
guangwu
💻 📖
wongearl
wongearl
💻		wenwenxiong
wenwenxiong
💻		柏喵Sakura
柏喵Sakura
💻		cui fliter
cui fliter
📖		刘旭
刘旭
💻
- - - - - diff --git a/api/v1beta1/auth_types.go b/api/v1beta1/auth_types.go deleted file mode 100644 index 1264deb2..00000000 --- a/api/v1beta1/auth_types.go +++ /dev/null @@ -1,54 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -import ( - "time" -) - -// Auth contains the SSH authentication configuration for machines. -type Auth struct { - // User is the username for SSH authentication. - // +optional - User string `yaml:"user,omitempty" json:"user,omitempty"` - - // Password is the password for SSH authentication. - // +optional - Password string `yaml:"password,omitempty" json:"password,omitempty"` - - // Port is the port for SSH authentication. - // +optional - Port *int `yaml:"port,omitempty" json:"port,omitempty"` - - // PrivateKey is the value of the private key for SSH authentication. - // +optional - PrivateKey string `yaml:"privateKey,omitempty" json:"privateKey,omitempty"` - - // PrivateKeyFile is the path to the private key for SSH authentication. - // +optional - PrivateKeyPath string `yaml:"privateKeyPath,omitempty" json:"privateKeyPath,omitempty"` - - // Secret is the secret of the PrivateKey or Password for SSH authentication.It should in the same namespace as capkk. - // When Password is empty, replace it with data.password. - // When PrivateKey is empty, replace it with data.privateKey - // +optional - Secret string `yaml:"secret,omitempty" json:"secret,omitempty"` - - // Timeout is the timeout for establish an SSH connection. - // +optional - Timeout *time.Duration `yaml:"timeout,omitempty" json:"timeout,omitempty"` -} diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go deleted file mode 100644 index 04310c2e..00000000 --- a/api/v1beta1/common_types.go +++ /dev/null @@ -1,22 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -const ( - // KKClusterLabelName is the label set on KKMachines and KKInstances linked to a kkCluster. - KKClusterLabelName = "kkcluster.infrastructure.cluster.x-k8s.io/cluster-name" -) diff --git a/api/v1beta1/component_types.go b/api/v1beta1/component_types.go deleted file mode 100644 index 36bbdffa..00000000 --- a/api/v1beta1/component_types.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -// Component is optional configuration for modifying the FTP server. -type Component struct { - // ZONE is the zone of the KKCluster where can get the binaries. - // If you have problem to access https://storage.googleapis.com, you can set "zone: cn". - // +optional - ZONE string `json:"zone,omitempty"` - - // Host is the host to download the binaries. - // +optional - Host string `json:"host,omitempty"` - - // Overrides is a list of components download information that need to be overridden. - // +optional - Overrides []Override `json:"overrides,omitempty"` -} - -// Override is a component download information that need to be overridden. -type Override struct { - // ID is the component id name. e.g. kubeadm, kubelet, containerd, etc. - ID string `json:"id,omitempty"` - - // Arch is the component arch. e.g. amd64, arm64, etc. - Arch string `json:"arch,omitempty"` - - // Version is the component version. e.g. v1.21.1, v1.22.0, etc. - Version string `json:"version,omitempty"` - - // URL is the download url of the binaries. - URL string `json:"url,omitempty"` - - // Path defines the URL path, which is the string of information that comes after the top level domain name. - Path string `json:"path,omitempty"` - - // Checksum is the SHA256 checksum of the binary. - // +optional - Checksum Checksum `json:"checksum,omitempty"` -} - -// Checksum is the SHA256 checksum of the binary. -type Checksum struct { - // Value is the checksum string value. - // +optional - Value string `json:"value,omitempty"` - - // Path defines the URL path, which is the path of the checksum file. - // +optional - Path string `json:"path,omitempty"` -} diff --git a/api/v1beta1/condition_consts.go b/api/v1beta1/condition_consts.go deleted file mode 100644 index 728ec876..00000000 --- a/api/v1beta1/condition_consts.go +++ /dev/null @@ -1,135 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -import ( - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" -) - -// KKCluster condition -const ( - // PrincipalPreparedCondition reports whether the principal is prepared. - PrincipalPreparedCondition clusterv1.ConditionType = "PrincipalPrepared" -) - -const ( - // HostReadyCondition reports whether the host is ready to be used. - HostReadyCondition clusterv1.ConditionType = "HostReadyCondition" -) - -const ( - // ExternalLoadBalancerReadyCondition reports on whether a control plane load balancer was successfully reconciled. - ExternalLoadBalancerReadyCondition clusterv1.ConditionType = "ExternalLoadBalancerReady" - - // WaitForDNSNameResolveReason used while waiting for DNS name to resolve. - WaitForDNSNameResolveReason = "WaitForDNSNameResolve" -) - -const ( - // CallKKInstanceInPlaceUpgradeCondition reports whether set up the InPlaceUpgradeVersionAnnotation annotation on all the KKInstance conditions. - CallKKInstanceInPlaceUpgradeCondition clusterv1.ConditionType = "CallKKInstanceInPlaceUpgrade" - - // KKInstanceObjectNotUpdatedReason used when the KKInstance is not updated. - KKInstanceObjectNotUpdatedReason = "KKInstanceObjectNotUpdated" - - // AllKKInstancesUpgradeCompletedCondition reports whether all the KKInstances are upgraded. - AllKKInstancesUpgradeCompletedCondition clusterv1.ConditionType = "AllKKInstancesUpgradeCompleted" - - // WaitingForKKInstancesUpgradeReason used when the KKCluster is waiting for all KKInstance to be upgrading completed. - WaitingForKKInstancesUpgradeReason = "WaitingForKKInstancesUpgrade" -) - -// KKMachine condition -const ( - // InstanceReadyCondition reports on current status of the SSH instance. Ready indicates the instance is in a Running state. - InstanceReadyCondition clusterv1.ConditionType = "InstanceReady" - - // InstanceNotFoundReason used when the instance couldn't be retrieved. - InstanceNotFoundReason = "InstanceNotFound" - // InstanceCleanedReason instance is in a Cleared state. - InstanceCleanedReason = "InstanceCleaned" - // InstanceNotReadyReason used when the instance is in a pending state. - InstanceNotReadyReason = "InstanceNotReady" - // InstanceInPlaceUpgradingReason used when the instance is in a InstanceStateInPlaceUpgrading state. - InstanceInPlaceUpgradingReason = "InstanceInPlaceUpgrading" - // InstanceBootstrapStartedReason set when the provisioning of an instance started. - InstanceBootstrapStartedReason = "InstanceBootstrapStarted" - // InstanceBootstrapFailedReason used for failures during instance provisioning. - InstanceBootstrapFailedReason = "InstanceBootstrapFailed" - // WaitingForClusterInfrastructureReason used when machine is waiting for cluster infrastructure to be ready before proceeding. - WaitingForClusterInfrastructureReason = "WaitingForClusterInfrastructure" - // WaitingForBootstrapDataReason used when machine is waiting for bootstrap data to be ready before proceeding. - WaitingForBootstrapDataReason = "WaitingForBootstrapData" -) - -// KKInstance condition -const ( - // KKInstanceBootstrappedCondition reports on current status of the instance. Ready indicates the instance is in a init Bootstrapped state. - KKInstanceBootstrappedCondition clusterv1.ConditionType = "InstanceBootstrapped" - // KKInstanceInitOSFailedReason used when the instance couldn't initialize os environment. - KKInstanceInitOSFailedReason = "InitOSFailed" -) - -const ( - // KKInstanceRepositoryReadyCondition reports on whether successful to use repository to install packages. - KKInstanceRepositoryReadyCondition clusterv1.ConditionType = "InstanceRepositoryReady" - // KKInstanceRepositoryFailedReason used when the instance couldn't use repository to install packages. - KKInstanceRepositoryFailedReason = "InstanceRepositoryFailed" -) - -const ( - // KKInstanceBinariesReadyCondition reports on whether successful to download binaries. - KKInstanceBinariesReadyCondition clusterv1.ConditionType = "InstanceBinariesReady" - // KKInstanceGetBinaryFailedReason used when the instance couldn't download binaries (or check existed binaries). - KKInstanceGetBinaryFailedReason = "GetBinaryFailed" -) - -const ( - // KKInstanceCRIReadyCondition reports on whether successful to download and install CRI. - KKInstanceCRIReadyCondition clusterv1.ConditionType = "InstanceCRIReady" - // KKInstanceInstallCRIFailedReason used when the instance couldn't download and install CRI. - KKInstanceInstallCRIFailedReason = "InstallCRIFailed" -) - -const ( - // KKInstanceProvisionedCondition reports on whether the instance is provisioned by cloud-init. - KKInstanceProvisionedCondition clusterv1.ConditionType = "InstanceProvisioned" - // KKInstanceRunCloudConfigFailedReason used when the instance couldn't be provisioned. - KKInstanceRunCloudConfigFailedReason = "RunCloudConfigFailed" -) - -const ( - // KKInstanceDeletingBootstrapCondition reports on whether the instance is deleting bootstrap data. - KKInstanceDeletingBootstrapCondition clusterv1.ConditionType = "InstanceDeletingBootstrapped" - // KKInstanceClearEnvironmentFailedReason used when the instance couldn't be deleting bootstrap data. - KKInstanceClearEnvironmentFailedReason = "ClearEnvironmentFailed" - - // CleaningReason (Severity=Info) documents a machine node being cleaned. - CleaningReason = "Cleaning" -) - -const ( - // KKInstanceInPlaceUpgradedCondition reports on whether the instance is in place upgrade succeed. - KKInstanceInPlaceUpgradedCondition clusterv1.ConditionType = "KKInstanceInPlaceUpgraded" -) - -const ( - // KKInstanceInPlaceUpgradeBinariesCondition reports on whether the instance is in place upgrade download binaries succeed. - KKInstanceInPlaceUpgradeBinariesCondition clusterv1.ConditionType = "KKInstanceInPlaceUpgradeBinaries" - // KKInstanceInPlaceGetBinaryFailedReason used when the instance couldn't download binaries (or check existed binaries). - KKInstanceInPlaceGetBinaryFailedReason = "KKInstanceInPlaceUpgradeGetBinaryFailed" -) diff --git a/api/v1beta1/containermanager_types.go b/api/v1beta1/containermanager_types.go deleted file mode 100644 index cdb2d1e4..00000000 --- a/api/v1beta1/containermanager_types.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -// Default values. -const ( - DockerType = "docker" - DefaultDockerVersion = "20.10.8" - DefaultCRIDockerdVersion = "0.2.6" - DefaultDockerCRISocket = "unix:///run/cri-dockerd.sock" - - ContainerdType = "containerd" - DefaultContainerdVersion = "1.6.4" - DefaultContainerdCRISocket = "unix:///var/run/containerd/containerd.sock" - - DefaultCrictlVersion = "v1.24.0" -) - -// ContainerManager defines the desired state of ContainerManager -type ContainerManager struct { - // CRISocket is used to connect an existing CRIClient. - // +optional - CRISocket string `json:"criSocket,omitempty"` - - // Type defines the type of ContainerManager. - // "docker", "containerd" - Type string `json:"type,omitempty"` - - // Version defines the version of ContainerManager. - Version string `json:"version,omitempty"` - - // CRICTLVersion defines the version of CRICTL. - CRICTLVersion string `json:"crictlVersion,omitempty"` - - // CRIDockerdVersion defines the version of cri-dockerd, available only when Type is docker. - // https://github.com/Mirantis/cri-dockerd - // +optional - CRIDockerdVersion string `json:"criDockerdVersion,omitempty"` -} diff --git a/api/v1beta1/doc.go b/api/v1beta1/doc.go deleted file mode 100644 index f8ef23a5..00000000 --- a/api/v1beta1/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package v1beta1 contains the v1beta1 API implementation. -package v1beta1 diff --git a/api/v1beta1/groupversion_info.go b/api/v1beta1/groupversion_info.go deleted file mode 100644 index 0e411b4a..00000000 --- a/api/v1beta1/groupversion_info.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package v1beta1 contains API Schema definitions for the infrastructure v1beta1 API group -// +kubebuilder:object:generate=true -// +groupName=infrastructure.cluster.x-k8s.io -package v1beta1 - -import ( - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/scheme" -) - -var ( - // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "infrastructure.cluster.x-k8s.io", Version: "v1beta1"} - - // SchemeBuilder is used to add go types to the GroupVersionKind scheme - SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} - - // AddToScheme adds the types in this group-version to the given scheme. - AddToScheme = SchemeBuilder.AddToScheme -) diff --git a/api/v1beta1/kkcluster_types.go b/api/v1beta1/kkcluster_types.go deleted file mode 100644 index ec7d936e..00000000 --- a/api/v1beta1/kkcluster_types.go +++ /dev/null @@ -1,191 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/errors" -) - -const ( - // ClusterFinalizer allows ReconcileKKCluster to clean up KK resources associated with KKCluster before - // removing it from the apiserver. - ClusterFinalizer = "kkcluster.infrastructure.cluster.x-k8s.io" - - // KUBERNETES the Kubernetes distributions - KUBERNETES = "kubernetes" - // K3S the K3S distributions - K3S = "k3s" - - // InPlaceUpgradeVersionAnnotation is the annotation that stores the version of the cluster used for in-place upgrade. - InPlaceUpgradeVersionAnnotation = "kkcluster.infrastructure.cluster.x-k8s.io/in-place-upgrade-version" -) - -// KKClusterSpec defines the desired state of KKCluster -type KKClusterSpec struct { - // Distribution represents the Kubernetes distribution type of the cluster. - Distribution string `json:"distribution,omitempty"` - - // Nodes represents the information about the nodes available to the cluster - Nodes Nodes `json:"nodes"` - - // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. - // +optional - ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` - - // ControlPlaneLoadBalancer is optional configuration for customizing control plane behavior. - ControlPlaneLoadBalancer *KKLoadBalancerSpec `json:"controlPlaneLoadBalancer,omitempty"` - - // Component is optional configuration for modifying the FTP server - // that gets the necessary components for the cluster. - // +optional - Component *Component `json:"component,omitempty"` - - // Registry represents the cluster image registry used to pull the images. - // +optional - Registry Registry `json:"registry,omitempty"` -} - -// Nodes represents the information about the nodes available to the cluster -type Nodes struct { - // Auth is the SSH authentication information of all instance. It is a global auth configuration. - // +optional - Auth Auth `json:"auth"` - - // Instances defines all instance contained in kkcluster. - Instances []InstanceInfo `json:"instances"` -} - -// InstanceInfo defines the information about the instance. -type InstanceInfo struct { - // Name is the host name of the machine. - // +kubebuilder:validation:MinLength=1 - Name string `json:"name,omitempty"` - - // Address is the IP address of the machine. - Address string `json:"address,omitempty"` - - // InternalAddress is the internal IP address of the machine. - InternalAddress string `json:"internalAddress,omitempty"` - - // Roles is the role of the machine. - // +optional - Roles []Role `json:"roles,omitempty"` - - // Arch is the architecture of the machine. e.g. "amd64", "arm64". - // +optional - Arch string `json:"arch,omitempty"` - - // Auth is the SSH authentication information of this machine. It will override the global auth configuration. - // +optional - Auth Auth `json:"auth,omitempty"` -} - -// KKLoadBalancerSpec defines the desired state of an KK load balancer. -type KKLoadBalancerSpec struct { - // The hostname on which the API server is serving. - Host string `json:"host,omitempty"` -} - -// KKClusterStatus defines the observed state of KKCluster -type KKClusterStatus struct { - // +kubebuilder:default=false - Ready bool `json:"ready"` - // FailureReason will be set in the event that there is a terminal problem - // reconciling the Machine and will contain a succinct value suitable - // for machine interpretation. - // - // This field should not be set for transitive errors that a controller - // faces that are expected to be fixed automatically over - // time (like service outages), but instead indicate that something is - // fundamentally wrong with the Machine's spec or the configuration of - // the controller, and that manual intervention is required. Examples - // of terminal errors would be invalid combinations of settings in the - // spec, values that are unsupported by the controller, or the - // responsible controller itself being critically misconfigured. - // - // Any transient errors that occur during the reconciliation of Machines - // can be added as events to the Machine object and/or logged in the - // controller's output. - // +optional - FailureReason *errors.MachineStatusError `json:"failureReason,omitempty"` - - // FailureMessage will be set in the event that there is a terminal problem - // reconciling the Machine and will contain a more verbose string suitable - // for logging and human consumption. - // - // This field should not be set for transitive errors that a controller - // faces that are expected to be fixed automatically over - // time (like service outages), but instead indicate that something is - // fundamentally wrong with the Machine's spec or the configuration of - // the controller, and that manual intervention is required. Examples - // of terminal errors would be invalid combinations of settings in the - // spec, values that are unsupported by the controller, or the - // responsible controller itself being critically misconfigured. - // - // Any transient errors that occur during the reconciliation of Machines - // can be added as events to the Machine object and/or logged in the - // controller's output. - // +optional - FailureMessage *string `json:"failureMessage,omitempty"` - - // Conditions defines current service state of the KKMachine. - // +optional - Conditions clusterv1.Conditions `json:"conditions,omitempty"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:resource:path=kkclusters,scope=Namespaced,categories=cluster-api,shortName=kkc -// +kubebuilder:storageversion -// +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels.cluster\\.x-k8s\\.io/cluster-name",description="Cluster to which this KKClusters belongs" -// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.ready",description="Cluster infrastructure is ready for SSH instances" -// +kubebuilder:printcolumn:name="Endpoint",type="string",JSONPath=".spec.controlPlaneEndpoint",description="API Endpoint",priority=1 -// +k8s:defaulter-gen=true - -// KKCluster is the Schema for the kkclusters API -type KKCluster struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec KKClusterSpec `json:"spec,omitempty"` - Status KKClusterStatus `json:"status,omitempty"` -} - -//+kubebuilder:object:root=true - -// KKClusterList contains a list of KKCluster -type KKClusterList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []KKCluster `json:"items"` -} - -// GetConditions returns the observations of the operational state of the KKCluster resource. -func (k *KKCluster) GetConditions() clusterv1.Conditions { - return k.Status.Conditions -} - -// SetConditions sets the underlying service state of the KKCluster to the predescribed clusterv1.Conditions. -func (k *KKCluster) SetConditions(conditions clusterv1.Conditions) { - k.Status.Conditions = conditions -} - -func init() { - SchemeBuilder.Register(&KKCluster{}, &KKClusterList{}) -} diff --git a/api/v1beta1/kkcluster_webhook.go b/api/v1beta1/kkcluster_webhook.go deleted file mode 100644 index 0af9fb58..00000000 --- a/api/v1beta1/kkcluster_webhook.go +++ /dev/null @@ -1,247 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "fmt" - "net" - "strings" - "time" - - mapset "github.com/deckarep/golang-set" - "github.com/google/go-cmp/cmp" - "github.com/pkg/errors" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/apiserver/pkg/storage/names" - "sigs.k8s.io/cluster-api/util/version" - ctrl "sigs.k8s.io/controller-runtime" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -const ( - defaultSSHUser = "root" - defaultSSHPort = 22 - defaultSSHEstablishTimeout = 30 * time.Second -) - -// log is for logging in this package. -var kkclusterlog = logf.Log.WithName("kkcluster-resource") - -func (k *KKCluster) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(k). - Complete() -} - -//+kubebuilder:webhook:path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkcluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkclusters,verbs=create;update,versions=v1beta1,name=default.kkcluster.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Defaulter = &KKCluster{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (k *KKCluster) Default() { - kkclusterlog.Info("default", "name", k.Name) - - defaultDistribution(&k.Spec) - defaultAuth(&k.Spec.Nodes.Auth) - defaultInstance(&k.Spec) - defaultInPlaceUpgradeAnnotation(k.GetAnnotations()) -} - -func defaultDistribution(spec *KKClusterSpec) { - if spec.Distribution == "" { - spec.Distribution = "kubernetes" - } - if spec.Distribution == "k8s" { - spec.Distribution = "kubernetes" - } -} - -func defaultAuth(auth *Auth) { - if auth.User == "" { - auth.User = defaultSSHUser - } - if auth.Port == nil { - p := defaultSSHPort - auth.Port = &p - } - if auth.Timeout == nil { - t := defaultSSHEstablishTimeout - auth.Timeout = &t - } -} - -func defaultInstance(spec *KKClusterSpec) { - for i := range spec.Nodes.Instances { - instance := &spec.Nodes.Instances[i] - if instance.Name == "" { - instance.Name = names.SimpleNameGenerator.GenerateName("kk-instance-") - } - if instance.InternalAddress == "" { - instance.InternalAddress = instance.Address - } - if instance.Arch == "" { - instance.Arch = "amd64" - } - if len(instance.Roles) == 0 { - instance.Roles = []Role{ControlPlane, Worker} - } - } -} - -func defaultInPlaceUpgradeAnnotation(annotation map[string]string) { - upgradeVersion, ok := annotation[InPlaceUpgradeVersionAnnotation] - if !ok { - return - } - - if !strings.HasPrefix(upgradeVersion, "v") { - annotation[InPlaceUpgradeVersionAnnotation] = "v" + upgradeVersion - } -} - -//+kubebuilder:webhook:path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-kkcluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkclusters,verbs=create;update,versions=v1beta1,name=validation.kkcluster.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Validator = &KKCluster{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (k *KKCluster) ValidateCreate() error { - kkclusterlog.Info("validate create", "name", k.Name) - - var allErrs field.ErrorList - allErrs = append(allErrs, validateDistribution(k.Spec)...) - allErrs = append(allErrs, validateClusterNodes(k.Spec.Nodes)...) - allErrs = append(allErrs, validateLoadBalancer(k.Spec.ControlPlaneLoadBalancer)...) - - return aggregateObjErrors(k.GroupVersionKind().GroupKind(), k.Name, allErrs) -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (k *KKCluster) ValidateUpdate(old runtime.Object) error { - kkclusterlog.Info("validate update", "name", k.Name) - - var allErrs field.ErrorList - oldC, ok := old.(*KKCluster) - if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected an KKCluster but got a %T", old)) - } - - newLoadBalancer := &KKLoadBalancerSpec{} - - if k.Spec.ControlPlaneLoadBalancer != nil { - newLoadBalancer = k.Spec.ControlPlaneLoadBalancer.DeepCopy() - } - - if oldC.Spec.ControlPlaneLoadBalancer != nil { - // If old scheme was not nil, the new scheme should be the same. - existingLoadBalancer := oldC.Spec.ControlPlaneLoadBalancer.DeepCopy() - if !cmp.Equal(newLoadBalancer, existingLoadBalancer) { - allErrs = append(allErrs, - field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer"), - k.Spec.ControlPlaneLoadBalancer, "field is immutable"), - ) - } - } - - allErrs = append(allErrs, validateClusterNodes(k.Spec.Nodes)...) - allErrs = append(allErrs, validateInPlaceUpgrade(k.GetAnnotations())...) - return aggregateObjErrors(k.GroupVersionKind().GroupKind(), k.Name, allErrs) -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (k *KKCluster) ValidateDelete() error { - return nil -} - -func validateDistribution(spec KKClusterSpec) []*field.Error { - var errs field.ErrorList - path := field.NewPath("spec", "distribution") - switch spec.Distribution { - case K3S: - return errs - case KUBERNETES: - return errs - default: - errs = append(errs, field.NotSupported(path, spec.Distribution, []string{K3S, KUBERNETES})) - } - return errs -} - -func validateLoadBalancer(loadBalancer *KKLoadBalancerSpec) []*field.Error { - var errs field.ErrorList - path := field.NewPath("spec", "controlPlaneLoadBalancer") - if loadBalancer.Host == "" { - errs = append(errs, field.Required(path.Child("host"), "can't be empty")) - } - return errs -} - -func validateClusterNodes(nodes Nodes) []*field.Error { - var errs field.ErrorList - - if nodes.Auth.Password == "" && nodes.Auth.PrivateKey == "" && nodes.Auth.PrivateKeyPath == "" && nodes.Auth.Secret == "" { - errs = append(errs, field.Required(field.NewPath("spec", "nodes", "auth"), "password and privateKey can't both be empty")) - } - - nameSet := mapset.NewThreadUnsafeSet() - addrSet := mapset.NewThreadUnsafeSet() - internalAddrSet := mapset.NewThreadUnsafeSet() - for i := range nodes.Instances { - instance := nodes.Instances[i] - path := field.NewPath("spec", "nodes", fmt.Sprintf("instances[%d]", i)) - if strings.ToLower(instance.Name) != instance.Name { - errs = append(errs, - field.Forbidden(path.Child("name"), "instance name must be the lower case")) - } - if !nameSet.Add(instance.Name) { - errs = append(errs, field.Duplicate(path.Child("name"), instance.Name)) - } - - if net.ParseIP(instance.Address) == nil { - errs = append(errs, field.Invalid(path.Child("address"), instance.Address, "instance address is invalid")) - } - if !addrSet.Add(instance.Address) { - errs = append(errs, field.Duplicate(path.Child("address"), instance.Address)) - } - - if net.ParseIP(instance.InternalAddress) == nil { - errs = append(errs, field.Invalid(path.Child("internalAddress"), instance.InternalAddress, "instance internalAddress is invalid")) - } - if !internalAddrSet.Add(instance.InternalAddress) { - errs = append(errs, field.Duplicate(path.Child("internalAddress"), instance.InternalAddress)) - } - } - return errs -} - -func validateInPlaceUpgrade(newAnnotation map[string]string) []*field.Error { - var allErrs field.ErrorList - - if v, ok := newAnnotation[InPlaceUpgradeVersionAnnotation]; ok { - _, err := version.ParseMajorMinorPatch(v) - if err != nil { - allErrs = append(allErrs, - field.InternalError( - field.NewPath("metadata", "annotations"), - errors.Wrapf(err, "failed to parse in-place upgrade version: %s", v)), - ) - } - } - return allErrs -} diff --git a/api/v1beta1/kkclustertemplate_types.go b/api/v1beta1/kkclustertemplate_types.go deleted file mode 100644 index a20f1228..00000000 --- a/api/v1beta1/kkclustertemplate_types.go +++ /dev/null @@ -1,63 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" -) - -// KKClusterTemplateSpec defines the desired state of KKClusterTemplate -type KKClusterTemplateSpec struct { - Template KKClusterTemplateResource `json:"template"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:resource:path=kkclustertemplates,scope=Namespaced,categories=cluster-api,shortName=kkct -// +kubebuilder:storageversion -// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of KKClusterTemplate" -// +k8s:defaulter-gen=true - -// KKClusterTemplate is the Schema for the kkclustertemplates API -type KKClusterTemplate struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec KKClusterTemplateSpec `json:"spec,omitempty"` -} - -//+kubebuilder:object:root=true - -// KKClusterTemplateList contains a list of KKClusterTemplate -type KKClusterTemplateList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []KKClusterTemplate `json:"items"` -} - -func init() { - SchemeBuilder.Register(&KKClusterTemplate{}, &KKClusterTemplateList{}) -} - -// KKClusterTemplateResource Standard object's metadata -type KKClusterTemplateResource struct { - // Standard object's metadata. - // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - // +optional - ObjectMeta clusterv1.ObjectMeta `json:"metadata,omitempty"` - Spec KKClusterSpec `json:"spec"` -} diff --git a/api/v1beta1/kkclustertemplate_webhook.go b/api/v1beta1/kkclustertemplate_webhook.go deleted file mode 100644 index ce365602..00000000 --- a/api/v1beta1/kkclustertemplate_webhook.go +++ /dev/null @@ -1,82 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "github.com/google/go-cmp/cmp" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" - ctrl "sigs.k8s.io/controller-runtime" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -// log is for logging in this package. -var kkclustertemplatelog = logf.Log.WithName("kkclustertemplate-resource") - -// SetupWebhookWithManager sets up and registers the webhook with the manager. -func (r *KKClusterTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(r). - Complete() -} - -//+kubebuilder:webhook:path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkclustertemplate,mutating=true,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkclustertemplates,verbs=create;update,versions=v1beta1,name=default.kkclustertemplate.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Defaulter = &KKClusterTemplate{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (r *KKClusterTemplate) Default() { - kkclustertemplatelog.Info("default", "name", r.Name) - - defaultDistribution(&r.Spec.Template.Spec) - defaultAuth(&r.Spec.Template.Spec.Nodes.Auth) - defaultInstance(&r.Spec.Template.Spec) -} - -//+kubebuilder:webhook:path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-kkclustertemplate,mutating=false,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkclustertemplates,verbs=create;update,versions=v1beta1,name=validation.kkclustertemplate.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Validator = &KKClusterTemplate{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (r *KKClusterTemplate) ValidateCreate() error { - kkclustertemplatelog.Info("validate create", "name", r.Name) - - var allErrs field.ErrorList - allErrs = append(allErrs, validateDistribution(r.Spec.Template.Spec)...) - allErrs = append(allErrs, validateClusterNodes(r.Spec.Template.Spec.Nodes)...) - allErrs = append(allErrs, validateLoadBalancer(r.Spec.Template.Spec.ControlPlaneLoadBalancer)...) - - return aggregateObjErrors(r.GroupVersionKind().GroupKind(), r.Name, allErrs) -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (r *KKClusterTemplate) ValidateUpdate(old runtime.Object) error { - kkclustertemplatelog.Info("validate update", "name", r.Name) - - oldC := old.(*KKClusterTemplate) - if !cmp.Equal(r.Spec, oldC.Spec) { - return apierrors.NewBadRequest("KKClusterTemplate.Spec is immutable") - } - return nil -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (r *KKClusterTemplate) ValidateDelete() error { - return nil -} diff --git a/api/v1beta1/kkinstance_types.go b/api/v1beta1/kkinstance_types.go deleted file mode 100644 index b495482b..00000000 --- a/api/v1beta1/kkinstance_types.go +++ /dev/null @@ -1,206 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/errors" -) - -const ( - // InstanceFinalizer allows ReconcileKKInstance to clean up KubeKey resources associated with KKInstance before - // removing it from the apiserver. - InstanceFinalizer = "kkinstance.infrastructure.cluster.x-k8s.io" -) - -// InstanceState describes the state of an KK instance. -type InstanceState string - -var ( - // InstanceStatePending is the string representing an instance in a pending state. - InstanceStatePending = InstanceState("pending") - - // InstanceStateBootstrapping is the string representing an instance in a bootstrapping state. - InstanceStateBootstrapping = InstanceState("bootstrapping") - - // InstanceStateBootstrapped = InstanceState("bootstrapped") - - // InstanceStateRunning is the string representing an instance in a running state. - InstanceStateRunning = InstanceState("running") - - // InstanceStateInPlaceUpgrading is the string representing an instance in an in-place-upgrading state. - InstanceStateInPlaceUpgrading = InstanceState("InPlaceUpgrading") - - // InstanceStateCleaning is the string representing an instance in a cleaning state. - InstanceStateCleaning = InstanceState("cleaning") - - // InstanceStateCleaned is the string representing an instance in a cleared state. - InstanceStateCleaned = InstanceState("cleaned") - - // InstanceRunningStates defines the set of states in which an SSH instance is - // running or going to be running soon. - InstanceRunningStates = sets.NewString( - string(InstanceStatePending), - string(InstanceStateRunning), - ) - - // InstanceKnownStates represents all known KKInstance states. - InstanceKnownStates = InstanceRunningStates.Union( - sets.NewString( - string(InstanceStateBootstrapping), - string(InstanceStateCleaning), - string(InstanceStateCleaned), - string(InstanceStateInPlaceUpgrading), - ), - ) -) - -// KKInstanceSpec defines the desired state of KKInstance -type KKInstanceSpec struct { - // Name is the host name of the machine. - // +kubebuilder:validation:MinLength=1 - Name string `json:"name,omitempty"` - - // Address is the IP address of the machine. - Address string `json:"address,omitempty"` - - // InternalAddress is the internal IP address of the machine. - InternalAddress string `json:"internalAddress,omitempty"` - - // Roles is the role of the machine. - // +optional - Roles []Role `json:"roles,omitempty"` - - // Arch is the architecture of the machine. e.g. "amd64", "arm64". - // +optional - Arch string `json:"arch,omitempty"` - - // Auth is the SSH authentication information of this machine. It will override the global auth configuration. - // +optional - Auth Auth `json:"auth,omitempty"` - - // ContainerManager is the container manager config of this machine. - // +optional - ContainerManager ContainerManager `json:"containerManager,omitempty"` - - // Repository is the repository config of this machine. - // +optional - Repository *Repository `json:"repository,omitempty"` -} - -// KKInstanceStatus defines the observed state of KKInstance -type KKInstanceStatus struct { - // The current state of the instance. - State InstanceState `json:"instanceState,omitempty"` - - // NodeRef will point to the corresponding Node if it exists. - // +optional - NodeRef *corev1.ObjectReference `json:"nodeRef,omitempty"` - - // NodeInfo is a set of ids/uuids to uniquely identify the node. - // More info: https://kubernetes.io/docs/concepts/nodes/node/#info - // +optional - NodeInfo *corev1.NodeSystemInfo `json:"nodeInfo,omitempty"` - - // FailureReason will be set in the event that there is a terminal problem - // reconciling the Machine and will contain a succinct value suitable - // for machine interpretation. - // - // This field should not be set for transitive errors that a controller - // faces that are expected to be fixed automatically over - // time (like service outages), but instead indicate that something is - // fundamentally wrong with the Machine's spec or the configuration of - // the controller, and that manual intervention is required. Examples - // of terminal errors would be invalid combinations of settings in the - // spec, values that are unsupported by the controller, or the - // responsible controller itself being critically misconfigured. - // - // Any transient errors that occur during the reconciliation of Machines - // can be added as events to the Machine object and/or logged in the - // controller's output. - // +optional - FailureReason *errors.MachineStatusError `json:"failureReason,omitempty"` - - // FailureMessage will be set in the event that there is a terminal problem - // reconciling the Machine and will contain a more verbose string suitable - // for logging and human consumption. - // - // This field should not be set for transitive errors that a controller - // faces that are expected to be fixed automatically over - // time (like service outages), but instead indicate that something is - // fundamentally wrong with the Machine's spec or the configuration of - // the controller, and that manual intervention is required. Examples - // of terminal errors would be invalid combinations of settings in the - // spec, values that are unsupported by the controller, or the - // responsible controller itself being critically misconfigured. - // - // Any transient errors that occur during the reconciliation of Machines - // can be added as events to the Machine object and/or logged in the - // controller's output. - // +optional - FailureMessage *string `json:"failureMessage,omitempty"` - - // Conditions defines current service state of the KKMachine. - // +optional - Conditions clusterv1.Conditions `json:"conditions,omitempty"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status -// +kubebuilder:resource:path=kkinstances,scope=Namespaced,categories=cluster-api,shortName=kki -// +kubebuilder:storageversion -// +kubebuilder:printcolumn:name="Hostname",type="string",JSONPath=".spec.name",description="kubekey instance hostname" -// +kubebuilder:printcolumn:name="Address",type="string",JSONPath=".spec.address",description="kubekey instance address" -// +kubebuilder:printcolumn:name="State",type="string",JSONPath=".status.instanceState",description="KKInstance state" -// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of KKInstance" -// +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".status.nodeInfo.kubeletVersion",description="Kubernetes version associated with this KKInstance" -// +k8s:defaulter-gen=true - -// KKInstance is the Schema for the kkinstances API -type KKInstance struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec KKInstanceSpec `json:"spec,omitempty"` - Status KKInstanceStatus `json:"status,omitempty"` -} - -// GetConditions returns the observations of the operational state of the KKMachine resource. -func (k *KKInstance) GetConditions() clusterv1.Conditions { - return k.Status.Conditions -} - -// SetConditions sets the underlying service state of the KKInstance to the predescribed clusterv1.Conditions. -func (k *KKInstance) SetConditions(conditions clusterv1.Conditions) { - k.Status.Conditions = conditions -} - -//+kubebuilder:object:root=true - -// KKInstanceList contains a list of KKInstance -type KKInstanceList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []KKInstance `json:"items"` -} - -func init() { - SchemeBuilder.Register(&KKInstance{}, &KKInstanceList{}) -} diff --git a/api/v1beta1/kkinstance_webhook.go b/api/v1beta1/kkinstance_webhook.go deleted file mode 100644 index f3d7b689..00000000 --- a/api/v1beta1/kkinstance_webhook.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" - "sigs.k8s.io/cluster-api/util/version" - ctrl "sigs.k8s.io/controller-runtime" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -// log is for logging in this package. -var kkinstancelog = logf.Log.WithName("kkinstance-resource") - -func (k *KKInstance) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(k). - Complete() -} - -//+kubebuilder:webhook:path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkinstance,mutating=true,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkinstances,verbs=create;update,versions=v1beta1,name=default.kkinstance.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Defaulter = &KKInstance{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (k *KKInstance) Default() { - kkinstancelog.Info("default", "name", k.Name) -} - -//+kubebuilder:webhook:path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-kkinstance,mutating=false,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkinstances,verbs=create;update,versions=v1beta1,name=validation.kkinstance.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Validator = &KKInstance{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (k *KKInstance) ValidateCreate() error { - kkinstancelog.Info("validate create", "name", k.Name) - return nil -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (k *KKInstance) ValidateUpdate(old runtime.Object) error { - kkinstancelog.Info("validate update", "name", k.Name) - - var allErrs field.ErrorList - if v, ok := k.GetAnnotations()[InPlaceUpgradeVersionAnnotation]; ok { - if k.Status.NodeInfo == nil { - allErrs = append(allErrs, - field.Invalid( - field.NewPath("status", "nodeInfo"), - k.Status.NodeInfo, - "nodeInfo is required for in-place upgrade")) - } - newSemverVersion, err := version.ParseMajorMinorPatch(v) - if err != nil { - allErrs = append(allErrs, - field.InternalError( - field.NewPath("metadata", "annotations"), - errors.Wrapf(err, "failed to parse in-place upgrade version: %s", v)), - ) - } - oldSemverVersion, err := version.ParseMajorMinorPatch(k.Status.NodeInfo.KubeletVersion) - if err != nil { - allErrs = append(allErrs, - field.InternalError( - field.NewPath("status", "nodeInfo", "kubeletVersion"), - errors.Wrapf(err, "failed to parse old version: %s", k.Status.NodeInfo.KubeletVersion)), - ) - } - - if newSemverVersion.Equals(oldSemverVersion) { - allErrs = append(allErrs, - field.Invalid( - field.NewPath("metadata", "annotations"), - v, - "new version must be different from old version"), - ) - } - - if !(newSemverVersion.GT(oldSemverVersion) && - newSemverVersion.Major == oldSemverVersion.Major && - (newSemverVersion.Minor == oldSemverVersion.Minor+1 || newSemverVersion.Minor == oldSemverVersion.Minor)) { - allErrs = append(allErrs, - field.Invalid(field.NewPath("metadata", "annotations"), - v, "Skipping MINOR versions when upgrading is unsupported.")) - } - } - - return aggregateObjErrors(k.GroupVersionKind().GroupKind(), k.Name, allErrs) -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (k *KKInstance) ValidateDelete() error { - kkinstancelog.Info("validate delete", "name", k.Name) - return nil -} diff --git a/api/v1beta1/kkmachine_types.go b/api/v1beta1/kkmachine_types.go deleted file mode 100644 index aabb7728..00000000 --- a/api/v1beta1/kkmachine_types.go +++ /dev/null @@ -1,148 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/errors" -) - -const ( - // MachineFinalizer allows ReconcileKKMachine to clean up KubeKey resources associated with KKMachine before - // removing it from the apiserver. - MachineFinalizer = "kkmachine.infrastructure.cluster.x-k8s.io" -) - -// KKMachineSpec defines the desired state of KKMachine -type KKMachineSpec struct { - // ProviderID is the unique identifier as specified by the kubekey provider. - ProviderID *string `json:"providerID,omitempty"` - - // InstanceID is the name of the KKInstance. - InstanceID *string `json:"instanceID,omitempty"` - - // Roles is the role of the machine. - // +optional - Roles []Role `json:"roles"` - - // ContainerManager is the container manager config of this machine. - // +optional - ContainerManager ContainerManager `json:"containerManager"` - - // Repository is the repository config of this machine. - // +optional - Repository *Repository `json:"repository,omitempty"` -} - -// KKMachineStatus defines the observed state of KKMachine -type KKMachineStatus struct { - // Ready is true when the provider resource is ready. - // +optional - Ready bool `json:"ready"` - - // Addresses contains the KK instance associated addresses. - Addresses []clusterv1.MachineAddress `json:"addresses,omitempty"` - - // InstanceState is the state of the KK instance for this machine. - // +optional - InstanceState *InstanceState `json:"instanceState,omitempty"` - - // FailureReason will be set in the event that there is a terminal problem - // reconciling the Machine and will contain a succinct value suitable - // for machine interpretation. - // - // This field should not be set for transitive errors that a controller - // faces that are expected to be fixed automatically over - // time (like service outages), but instead indicate that something is - // fundamentally wrong with the Machine's spec or the configuration of - // the controller, and that manual intervention is required. Examples - // of terminal errors would be invalid combinations of settings in the - // spec, values that are unsupported by the controller, or the - // responsible controller itself being critically misconfigured. - // - // Any transient errors that occur during the reconciliation of Machines - // can be added as events to the Machine object and/or logged in the - // controller's output. - // +optional - FailureReason *errors.MachineStatusError `json:"failureReason,omitempty"` - - // FailureMessage will be set in the event that there is a terminal problem - // reconciling the Machine and will contain a more verbose string suitable - // for logging and human consumption. - // - // This field should not be set for transitive errors that a controller - // faces that are expected to be fixed automatically over - // time (like service outages), but instead indicate that something is - // fundamentally wrong with the Machine's spec or the configuration of - // the controller, and that manual intervention is required. Examples - // of terminal errors would be invalid combinations of settings in the - // spec, values that are unsupported by the controller, or the - // responsible controller itself being critically misconfigured. - // - // Any transient errors that occur during the reconciliation of Machines - // can be added as events to the Machine object and/or logged in the - // controller's output. - // +optional - FailureMessage *string `json:"failureMessage,omitempty"` - - // Conditions defines current service state of the KKMachine. - // +optional - Conditions clusterv1.Conditions `json:"conditions,omitempty"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:resource:path=kkmachines,scope=Namespaced,categories=cluster-api,shortName=kkm -// +kubebuilder:storageversion -// +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels.cluster\\.x-k8s\\.io/cluster-name",description="Cluster to which this KKMachine belongs" -// +kubebuilder:printcolumn:name="Instance",type="string",JSONPath=".spec.instanceID",description="KKInstance name" -// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.ready",description="Machine ready status" -// +kubebuilder:printcolumn:name="Machine",type="string",JSONPath=".metadata.ownerReferences[?(@.kind==\"Machine\")].name",description="Machine object which owns with this KKMachine" -// +k8s:defaulter-gen=true - -// KKMachine is the Schema for the kkmachines API -type KKMachine struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec KKMachineSpec `json:"spec,omitempty"` - Status KKMachineStatus `json:"status,omitempty"` -} - -// GetConditions returns the observations of the operational state of the KKMachine resource. -func (k *KKMachine) GetConditions() clusterv1.Conditions { - return k.Status.Conditions -} - -// SetConditions sets the underlying service state of the KKMachine to the predescribed clusterv1.Conditions. -func (k *KKMachine) SetConditions(conditions clusterv1.Conditions) { - k.Status.Conditions = conditions -} - -//+kubebuilder:object:root=true - -// KKMachineList contains a list of KKMachine -type KKMachineList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []KKMachine `json:"items"` -} - -func init() { - SchemeBuilder.Register(&KKMachine{}, &KKMachineList{}) -} diff --git a/api/v1beta1/kkmachine_webhook.go b/api/v1beta1/kkmachine_webhook.go deleted file mode 100644 index ff9b6e24..00000000 --- a/api/v1beta1/kkmachine_webhook.go +++ /dev/null @@ -1,119 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "strings" - - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" - ctrl "sigs.k8s.io/controller-runtime" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -// log is for logging in this package. -var kkmachinelog = logf.Log.WithName("kkmachine-resource") - -func (k *KKMachine) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(k). - Complete() -} - -//+kubebuilder:webhook:path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkmachine,mutating=true,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkmachines,verbs=create;update,versions=v1beta1,name=default.kkmachine.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Defaulter = &KKMachine{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (k *KKMachine) Default() { - kkmachinelog.Info("default", "name", k.Name) - - defaultContainerManager(&k.Spec) -} - -func defaultContainerManager(spec *KKMachineSpec) { - // Direct connection to the user-provided CRI socket - if spec.ContainerManager.CRISocket != "" { - return - } - - if spec.ContainerManager.Type == "" { - spec.ContainerManager.Type = ContainerdType - } - - switch spec.ContainerManager.Type { - case ContainerdType: - if spec.ContainerManager.Version == "" { - spec.ContainerManager.Version = DefaultContainerdVersion - } - spec.ContainerManager.CRISocket = DefaultContainerdCRISocket - case DockerType: - if spec.ContainerManager.Version == "" { - spec.ContainerManager.Version = DefaultDockerVersion - } - if spec.ContainerManager.CRIDockerdVersion == "" { - spec.ContainerManager.CRIDockerdVersion = DefaultCRIDockerdVersion - } - spec.ContainerManager.CRISocket = DefaultDockerCRISocket - } - - if spec.ContainerManager.CRICTLVersion == "" { - spec.ContainerManager.CRICTLVersion = DefaultCrictlVersion - } - - spec.ContainerManager.Version = strings.TrimPrefix(spec.ContainerManager.Version, "v") - - if !strings.HasPrefix(spec.ContainerManager.CRICTLVersion, "v") { - spec.ContainerManager.CRICTLVersion = "v" + spec.ContainerManager.CRICTLVersion - } -} - -//+kubebuilder:webhook:path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-kkmachine,mutating=false,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkmachines,verbs=create;update,versions=v1beta1,name=validation.kkmachine.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Validator = &KKMachine{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (k *KKMachine) ValidateCreate() error { - kkmachinelog.Info("validate create", "name", k.Name) - - var allErrs field.ErrorList - allErrs = append(allErrs, validateRepository(k.Spec.Repository)...) - return aggregateObjErrors(k.GroupVersionKind().GroupKind(), k.Name, allErrs) -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (k *KKMachine) ValidateUpdate(old runtime.Object) error { - kkmachinelog.Info("validate update", "name", k.Name) - var allErrs field.ErrorList - allErrs = append(allErrs, validateRepository(k.Spec.Repository)...) - return aggregateObjErrors(k.GroupVersionKind().GroupKind(), k.Name, allErrs) -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (k *KKMachine) ValidateDelete() error { - kkmachinelog.Info("validate delete", "name", k.Name) - return nil -} - -func validateRepository(repo *Repository) field.ErrorList { //nolint:unparam - var allErrs field.ErrorList - if repo == nil { - return allErrs - } - return allErrs -} diff --git a/api/v1beta1/kkmachine_webhook_test.go b/api/v1beta1/kkmachine_webhook_test.go deleted file mode 100644 index aebbd8ed..00000000 --- a/api/v1beta1/kkmachine_webhook_test.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -import ( - "testing" - - . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utildefaulting "sigs.k8s.io/cluster-api/util/defaulting" -) - -func TestKKMachine_Default(t *testing.T) { - g := NewWithT(t) - kkm := &KKMachine{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "foobar", - }, - } - - t.Run("for KKMachine", utildefaulting.DefaultValidateTest(kkm)) - kkm.Default() - - g.Expect(kkm.Spec.ContainerManager.Type).To(Equal(ContainerdType)) - g.Expect(kkm.Spec.ContainerManager.Version).To(Equal(DefaultContainerdVersion)) - g.Expect(kkm.Spec.ContainerManager.CRICTLVersion).To(Equal(DefaultCrictlVersion)) - - kkm2 := &KKMachine{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "foobar", - }, - Spec: KKMachineSpec{ - ContainerManager: ContainerManager{ - Type: ContainerdType, - Version: "v1.6.4", - CRICTLVersion: "1.24.0", - }, - }, - } - - t.Run("for KKMachine2", utildefaulting.DefaultValidateTest(kkm2)) - kkm2.Default() - - g.Expect(kkm2.Spec.ContainerManager.Type).To(Equal(ContainerdType)) - g.Expect(kkm2.Spec.ContainerManager.Version).To(Equal("1.6.4")) - g.Expect(kkm2.Spec.ContainerManager.CRICTLVersion).To(Equal("v1.24.0")) -} diff --git a/api/v1beta1/kkmachinetemplate_types.go b/api/v1beta1/kkmachinetemplate_types.go deleted file mode 100644 index 150dd8ae..00000000 --- a/api/v1beta1/kkmachinetemplate_types.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" -) - -// KKMachineTemplateStatus defines a status for an KKMachineTemplate. -type KKMachineTemplateStatus struct { - // Capacity defines the resource capacity for this machine. - // This value is used for autoscaling from zero operations as defined in: - // https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20210310-opt-in-autoscaling-from-zero.md - // +optional - Capacity corev1.ResourceList `json:"capacity,omitempty"` -} - -// KKMachineTemplateSpec defines the desired state of KKMachineTemplate -type KKMachineTemplateSpec struct { - Template KKMachineTemplateResource `json:"template"` -} - -// KKMachineTemplateResource describes the data needed to create am KKMachine from a template. -type KKMachineTemplateResource struct { - // Standard object's metadata. - // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - // +optional - ObjectMeta clusterv1.ObjectMeta `json:"metadata,omitempty"` - - // Spec is the specification of the desired behavior of the machine. - Spec KKMachineSpec `json:"spec"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:resource:path=kkmachinetemplates,scope=Namespaced,categories=cluster-api,shortName=kkmt -// +kubebuilder:storageversion -// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of KKMachineTemplate" -// +k8s:defaulter-gen=true - -// KKMachineTemplate is the Schema for the kkmachinetemplates API -type KKMachineTemplate struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec KKMachineTemplateSpec `json:"spec,omitempty"` - Status KKMachineTemplateStatus `json:"status,omitempty"` -} - -//+kubebuilder:object:root=true - -// KKMachineTemplateList contains a list of KKMachineTemplate -type KKMachineTemplateList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []KKMachineTemplate `json:"items"` -} - -func init() { - SchemeBuilder.Register(&KKMachineTemplate{}, &KKMachineTemplateList{}) -} diff --git a/api/v1beta1/kkmachinetemplate_webhook.go b/api/v1beta1/kkmachinetemplate_webhook.go deleted file mode 100644 index ae914285..00000000 --- a/api/v1beta1/kkmachinetemplate_webhook.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" - ctrl "sigs.k8s.io/controller-runtime" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -// log is for logging in this package. -var kkmachinetemplatelog = logf.Log.WithName("kkmachinetemplate-resource") - -func (k *KKMachineTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(k). - Complete() -} - -//+kubebuilder:webhook:path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkmachinetemplate,mutating=true,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkmachinetemplates,verbs=create;update,versions=v1beta1,name=default.kkmachinetemplate.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Defaulter = &KKMachineTemplate{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (k *KKMachineTemplate) Default() { - kkmachinetemplatelog.Info("default", "name", k.Name) - - defaultContainerManager(&k.Spec.Template.Spec) -} - -//+kubebuilder:webhook:path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-kkmachinetemplate,mutating=false,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=kkmachinetemplates,verbs=create;update,versions=v1beta1,name=validation.kkmachinetemplate.infrastructure.cluster.x-k8s.io,admissionReviewVersions=v1 - -var _ webhook.Validator = &KKMachineTemplate{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (k *KKMachineTemplate) ValidateCreate() error { - kkmachinetemplatelog.Info("validate create", "name", k.Name) - - spec := k.Spec.Template.Spec - var allErrs field.ErrorList - allErrs = append(allErrs, validateRepository(spec.Repository)...) - return aggregateObjErrors(k.GroupVersionKind().GroupKind(), k.Name, allErrs) -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (k *KKMachineTemplate) ValidateUpdate(old runtime.Object) error { - kkmachinetemplatelog.Info("validate update", "name", k.Name) - - spec := k.Spec.Template.Spec - var allErrs field.ErrorList - allErrs = append(allErrs, validateRepository(spec.Repository)...) - return aggregateObjErrors(k.GroupVersionKind().GroupKind(), k.Name, allErrs) -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (k *KKMachineTemplate) ValidateDelete() error { - kkmachinetemplatelog.Info("validate delete", "name", k.Name) - - return nil -} diff --git a/api/v1beta1/registry_types.go b/api/v1beta1/registry_types.go deleted file mode 100644 index 801e616f..00000000 --- a/api/v1beta1/registry_types.go +++ /dev/null @@ -1,68 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// Registry is the configuration for a cluster registry -type Registry struct { - metav1.TypeMeta `json:",inline"` - - // PrivateRegistry defines the private registry address of ContainerManager. - PrivateRegistry string `json:"privateRegistry"` - - // InsecureRegistries defines the insecure registries of ContainerManager. - InsecureRegistries []string `json:"insecureRegistries,omitempty"` - - // RegistryMirrors defines the registry mirrors of this PrivateRegistry. - RegistryMirrors []string `json:"registryMirrors,omitempty"` - - // NamespaceOverride defines the namespace override of this PrivateRegistry. - NamespaceOverride string `json:"namespaceOverride"` - - // Auth defines the auth of this PrivateRegistry. - Auth RegistryAuth `json:"auth"` -} - -// RegistryAuth defines the auth of a registry -type RegistryAuth struct { - // Username defines the username of this PrivateRegistry. - Username string `json:"username"` - - // Password defines the password of this PrivateRegistry. - Password string `json:"password"` - - // InsecureSkipVerify allow contacting this PrivateRegistry over HTTPS with failed TLS verification. - InsecureSkipVerify bool `json:"insecureSkipVerify"` - - // PlainHTTP allow contacting this PrivateRegistry over HTTP. - PlainHTTP bool `json:"plainHTTP"` - - // CertsPath defines the path of the certs files of this PrivateRegistry. - CertsPath string `json:"certsPath"` - - // CAFile is an SSL Certificate Authority file used to secure etcd communication. - CAFile string `yaml:"caFile" json:"caFile,omitempty"` - // CertFile is an SSL certification file used to secure etcd communication. - CertFile string `yaml:"certFile" json:"certFile,omitempty"` - // KeyFile is an SSL key file used to secure etcd communication. - KeyFile string `yaml:"keyFile" json:"keyFile,omitempty"` -} diff --git a/api/v1beta1/repository_types.go b/api/v1beta1/repository_types.go deleted file mode 100644 index 92e4bcb8..00000000 --- a/api/v1beta1/repository_types.go +++ /dev/null @@ -1,42 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -// ISO type -const ( - NONE = "none" - AUTO = "auto" -) - -// Repository defines the repository of the instance. -type Repository struct { - // ISO specifies the ISO file name. There are 3 options: - // "": empty string means will not install the packages. - // "none": no ISO file will be used. And capkk will use the default repository to install the required packages. - // "auto": capkk will detect the ISO file automatically. Only support Ubuntu/Debian/CentOS. - // "xxx-20.04-debs-amd64.iso": use the specified name to get the ISO file name. - // +optional - ISO string `json:"iso,omitempty"` - - // Update will update the repository packages list and cache if it is true. - // +optional - Update bool `json:"update,omitempty"` - - // Packages is a list of packages to be installed. - // +optional - Packages []string `json:"packages,omitempty"` -} diff --git a/api/v1beta1/role_types.go b/api/v1beta1/role_types.go deleted file mode 100644 index 7c8e6439..00000000 --- a/api/v1beta1/role_types.go +++ /dev/null @@ -1,27 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -// Role represents a role of a node. -type Role string - -// Internal roles. -const ( - ControlPlane Role = "control-plane" - Master Role = "master" - Worker Role = "worker" -) diff --git a/api/v1beta1/webhook.go b/api/v1beta1/webhook.go deleted file mode 100644 index 80f26b58..00000000 --- a/api/v1beta1/webhook.go +++ /dev/null @@ -1,35 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -import ( - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/validation/field" -) - -func aggregateObjErrors(gk schema.GroupKind, name string, allErrs field.ErrorList) error { - if len(allErrs) == 0 { - return nil - } - - return apierrors.NewInvalid( - gk, - name, - allErrs, - ) -} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go deleted file mode 100644 index f6dfd633..00000000 --- a/api/v1beta1/zz_generated.deepcopy.go +++ /dev/null @@ -1,839 +0,0 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by controller-gen. DO NOT EDIT. - -package v1beta1 - -import ( - "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - apiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/errors" - timex "time" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Auth) DeepCopyInto(out *Auth) { - *out = *in - if in.Port != nil { - in, out := &in.Port, &out.Port - *out = new(int) - **out = **in - } - if in.Timeout != nil { - in, out := &in.Timeout, &out.Timeout - *out = new(timex.Duration) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Auth. -func (in *Auth) DeepCopy() *Auth { - if in == nil { - return nil - } - out := new(Auth) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Checksum) DeepCopyInto(out *Checksum) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Checksum. -func (in *Checksum) DeepCopy() *Checksum { - if in == nil { - return nil - } - out := new(Checksum) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Component) DeepCopyInto(out *Component) { - *out = *in - if in.Overrides != nil { - in, out := &in.Overrides, &out.Overrides - *out = make([]Override, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Component. -func (in *Component) DeepCopy() *Component { - if in == nil { - return nil - } - out := new(Component) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ContainerManager) DeepCopyInto(out *ContainerManager) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerManager. -func (in *ContainerManager) DeepCopy() *ContainerManager { - if in == nil { - return nil - } - out := new(ContainerManager) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *InstanceInfo) DeepCopyInto(out *InstanceInfo) { - *out = *in - if in.Roles != nil { - in, out := &in.Roles, &out.Roles - *out = make([]Role, len(*in)) - copy(*out, *in) - } - in.Auth.DeepCopyInto(&out.Auth) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstanceInfo. -func (in *InstanceInfo) DeepCopy() *InstanceInfo { - if in == nil { - return nil - } - out := new(InstanceInfo) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKCluster) DeepCopyInto(out *KKCluster) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKCluster. -func (in *KKCluster) DeepCopy() *KKCluster { - if in == nil { - return nil - } - out := new(KKCluster) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKCluster) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKClusterList) DeepCopyInto(out *KKClusterList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]KKCluster, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterList. -func (in *KKClusterList) DeepCopy() *KKClusterList { - if in == nil { - return nil - } - out := new(KKClusterList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKClusterList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKClusterSpec) DeepCopyInto(out *KKClusterSpec) { - *out = *in - in.Nodes.DeepCopyInto(&out.Nodes) - out.ControlPlaneEndpoint = in.ControlPlaneEndpoint - if in.ControlPlaneLoadBalancer != nil { - in, out := &in.ControlPlaneLoadBalancer, &out.ControlPlaneLoadBalancer - *out = new(KKLoadBalancerSpec) - **out = **in - } - if in.Component != nil { - in, out := &in.Component, &out.Component - *out = new(Component) - (*in).DeepCopyInto(*out) - } - in.Registry.DeepCopyInto(&out.Registry) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterSpec. -func (in *KKClusterSpec) DeepCopy() *KKClusterSpec { - if in == nil { - return nil - } - out := new(KKClusterSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKClusterStatus) DeepCopyInto(out *KKClusterStatus) { - *out = *in - if in.FailureReason != nil { - in, out := &in.FailureReason, &out.FailureReason - *out = new(errors.MachineStatusError) - **out = **in - } - if in.FailureMessage != nil { - in, out := &in.FailureMessage, &out.FailureMessage - *out = new(string) - **out = **in - } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make(apiv1beta1.Conditions, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterStatus. -func (in *KKClusterStatus) DeepCopy() *KKClusterStatus { - if in == nil { - return nil - } - out := new(KKClusterStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKClusterTemplate) DeepCopyInto(out *KKClusterTemplate) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterTemplate. -func (in *KKClusterTemplate) DeepCopy() *KKClusterTemplate { - if in == nil { - return nil - } - out := new(KKClusterTemplate) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKClusterTemplate) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKClusterTemplateList) DeepCopyInto(out *KKClusterTemplateList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]KKClusterTemplate, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterTemplateList. -func (in *KKClusterTemplateList) DeepCopy() *KKClusterTemplateList { - if in == nil { - return nil - } - out := new(KKClusterTemplateList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKClusterTemplateList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKClusterTemplateResource) DeepCopyInto(out *KKClusterTemplateResource) { - *out = *in - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterTemplateResource. -func (in *KKClusterTemplateResource) DeepCopy() *KKClusterTemplateResource { - if in == nil { - return nil - } - out := new(KKClusterTemplateResource) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKClusterTemplateSpec) DeepCopyInto(out *KKClusterTemplateSpec) { - *out = *in - in.Template.DeepCopyInto(&out.Template) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKClusterTemplateSpec. -func (in *KKClusterTemplateSpec) DeepCopy() *KKClusterTemplateSpec { - if in == nil { - return nil - } - out := new(KKClusterTemplateSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKInstance) DeepCopyInto(out *KKInstance) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKInstance. -func (in *KKInstance) DeepCopy() *KKInstance { - if in == nil { - return nil - } - out := new(KKInstance) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKInstance) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKInstanceList) DeepCopyInto(out *KKInstanceList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]KKInstance, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKInstanceList. -func (in *KKInstanceList) DeepCopy() *KKInstanceList { - if in == nil { - return nil - } - out := new(KKInstanceList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKInstanceList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKInstanceSpec) DeepCopyInto(out *KKInstanceSpec) { - *out = *in - if in.Roles != nil { - in, out := &in.Roles, &out.Roles - *out = make([]Role, len(*in)) - copy(*out, *in) - } - in.Auth.DeepCopyInto(&out.Auth) - out.ContainerManager = in.ContainerManager - if in.Repository != nil { - in, out := &in.Repository, &out.Repository - *out = new(Repository) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKInstanceSpec. -func (in *KKInstanceSpec) DeepCopy() *KKInstanceSpec { - if in == nil { - return nil - } - out := new(KKInstanceSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKInstanceStatus) DeepCopyInto(out *KKInstanceStatus) { - *out = *in - if in.NodeRef != nil { - in, out := &in.NodeRef, &out.NodeRef - *out = new(v1.ObjectReference) - **out = **in - } - if in.NodeInfo != nil { - in, out := &in.NodeInfo, &out.NodeInfo - *out = new(v1.NodeSystemInfo) - **out = **in - } - if in.FailureReason != nil { - in, out := &in.FailureReason, &out.FailureReason - *out = new(errors.MachineStatusError) - **out = **in - } - if in.FailureMessage != nil { - in, out := &in.FailureMessage, &out.FailureMessage - *out = new(string) - **out = **in - } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make(apiv1beta1.Conditions, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKInstanceStatus. -func (in *KKInstanceStatus) DeepCopy() *KKInstanceStatus { - if in == nil { - return nil - } - out := new(KKInstanceStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKLoadBalancerSpec) DeepCopyInto(out *KKLoadBalancerSpec) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKLoadBalancerSpec. -func (in *KKLoadBalancerSpec) DeepCopy() *KKLoadBalancerSpec { - if in == nil { - return nil - } - out := new(KKLoadBalancerSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachine) DeepCopyInto(out *KKMachine) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachine. -func (in *KKMachine) DeepCopy() *KKMachine { - if in == nil { - return nil - } - out := new(KKMachine) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKMachine) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachineList) DeepCopyInto(out *KKMachineList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]KKMachine, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineList. -func (in *KKMachineList) DeepCopy() *KKMachineList { - if in == nil { - return nil - } - out := new(KKMachineList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKMachineList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachineSpec) DeepCopyInto(out *KKMachineSpec) { - *out = *in - if in.ProviderID != nil { - in, out := &in.ProviderID, &out.ProviderID - *out = new(string) - **out = **in - } - if in.InstanceID != nil { - in, out := &in.InstanceID, &out.InstanceID - *out = new(string) - **out = **in - } - if in.Roles != nil { - in, out := &in.Roles, &out.Roles - *out = make([]Role, len(*in)) - copy(*out, *in) - } - out.ContainerManager = in.ContainerManager - if in.Repository != nil { - in, out := &in.Repository, &out.Repository - *out = new(Repository) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineSpec. -func (in *KKMachineSpec) DeepCopy() *KKMachineSpec { - if in == nil { - return nil - } - out := new(KKMachineSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachineStatus) DeepCopyInto(out *KKMachineStatus) { - *out = *in - if in.Addresses != nil { - in, out := &in.Addresses, &out.Addresses - *out = make([]apiv1beta1.MachineAddress, len(*in)) - copy(*out, *in) - } - if in.InstanceState != nil { - in, out := &in.InstanceState, &out.InstanceState - *out = new(InstanceState) - **out = **in - } - if in.FailureReason != nil { - in, out := &in.FailureReason, &out.FailureReason - *out = new(errors.MachineStatusError) - **out = **in - } - if in.FailureMessage != nil { - in, out := &in.FailureMessage, &out.FailureMessage - *out = new(string) - **out = **in - } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make(apiv1beta1.Conditions, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineStatus. -func (in *KKMachineStatus) DeepCopy() *KKMachineStatus { - if in == nil { - return nil - } - out := new(KKMachineStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachineTemplate) DeepCopyInto(out *KKMachineTemplate) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineTemplate. -func (in *KKMachineTemplate) DeepCopy() *KKMachineTemplate { - if in == nil { - return nil - } - out := new(KKMachineTemplate) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKMachineTemplate) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachineTemplateList) DeepCopyInto(out *KKMachineTemplateList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]KKMachineTemplate, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineTemplateList. -func (in *KKMachineTemplateList) DeepCopy() *KKMachineTemplateList { - if in == nil { - return nil - } - out := new(KKMachineTemplateList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *KKMachineTemplateList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachineTemplateResource) DeepCopyInto(out *KKMachineTemplateResource) { - *out = *in - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineTemplateResource. -func (in *KKMachineTemplateResource) DeepCopy() *KKMachineTemplateResource { - if in == nil { - return nil - } - out := new(KKMachineTemplateResource) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachineTemplateSpec) DeepCopyInto(out *KKMachineTemplateSpec) { - *out = *in - in.Template.DeepCopyInto(&out.Template) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineTemplateSpec. -func (in *KKMachineTemplateSpec) DeepCopy() *KKMachineTemplateSpec { - if in == nil { - return nil - } - out := new(KKMachineTemplateSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KKMachineTemplateStatus) DeepCopyInto(out *KKMachineTemplateStatus) { - *out = *in - if in.Capacity != nil { - in, out := &in.Capacity, &out.Capacity - *out = make(v1.ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KKMachineTemplateStatus. -func (in *KKMachineTemplateStatus) DeepCopy() *KKMachineTemplateStatus { - if in == nil { - return nil - } - out := new(KKMachineTemplateStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Nodes) DeepCopyInto(out *Nodes) { - *out = *in - in.Auth.DeepCopyInto(&out.Auth) - if in.Instances != nil { - in, out := &in.Instances, &out.Instances - *out = make([]InstanceInfo, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Nodes. -func (in *Nodes) DeepCopy() *Nodes { - if in == nil { - return nil - } - out := new(Nodes) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Override) DeepCopyInto(out *Override) { - *out = *in - out.Checksum = in.Checksum -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Override. -func (in *Override) DeepCopy() *Override { - if in == nil { - return nil - } - out := new(Override) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Registry) DeepCopyInto(out *Registry) { - *out = *in - out.TypeMeta = in.TypeMeta - if in.InsecureRegistries != nil { - in, out := &in.InsecureRegistries, &out.InsecureRegistries - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.RegistryMirrors != nil { - in, out := &in.RegistryMirrors, &out.RegistryMirrors - *out = make([]string, len(*in)) - copy(*out, *in) - } - out.Auth = in.Auth -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Registry. -func (in *Registry) DeepCopy() *Registry { - if in == nil { - return nil - } - out := new(Registry) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Registry) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RegistryAuth) DeepCopyInto(out *RegistryAuth) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RegistryAuth. -func (in *RegistryAuth) DeepCopy() *RegistryAuth { - if in == nil { - return nil - } - out := new(RegistryAuth) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Repository) DeepCopyInto(out *Repository) { - *out = *in - if in.Packages != nil { - in, out := &in.Packages, &out.Packages - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Repository. -func (in *Repository) DeepCopy() *Repository { - if in == nil { - return nil - } - out := new(Repository) - in.DeepCopyInto(out) - return out -} diff --git a/bootstrap/k3s/PROJECT b/bootstrap/k3s/PROJECT deleted file mode 100644 index f9bf80aa..00000000 --- a/bootstrap/k3s/PROJECT +++ /dev/null @@ -1,35 +0,0 @@ -domain: cluster.x-k8s.io -layout: -- go.kubebuilder.io/v3 -plugins: - manifests.sdk.operatorframework.io/v2: {} - scorecard.sdk.operatorframework.io/v2: {} -projectName: cluster-api-bootstrap-provider-kubekey-k3s -repo: github.com/kubesphere/kubekey/bootstrap/k3s -resources: -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: cluster.x-k8s.io - group: bootstrap - kind: K3sConfig - path: github.com/kubesphere/kubekey/bootstrap/k3s/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -- api: - crdVersion: v1 - namespaced: true - domain: cluster.x-k8s.io - group: bootstrap - kind: K3sConfigTemplate - path: github.com/kubesphere/kubekey/bootstrap/k3s/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -version: "3" diff --git a/bootstrap/k3s/api/v1beta1/groupversion_info.go b/bootstrap/k3s/api/v1beta1/groupversion_info.go deleted file mode 100644 index 3c387174..00000000 --- a/bootstrap/k3s/api/v1beta1/groupversion_info.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package v1beta1 contains API Schema definitions for the bootstrap v1beta1 API group -// +kubebuilder:object:generate=true -// +groupName=bootstrap.cluster.x-k8s.io -package v1beta1 - -import ( - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/scheme" -) - -var ( - // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "bootstrap.cluster.x-k8s.io", Version: "v1beta1"} - - // SchemeBuilder is used to add go types to the GroupVersionKind scheme - SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} - - // AddToScheme adds the types in this group-version to the given scheme. - AddToScheme = SchemeBuilder.AddToScheme -) diff --git a/bootstrap/k3s/api/v1beta1/k3sconfig.go b/bootstrap/k3s/api/v1beta1/k3sconfig.go deleted file mode 100644 index 2917085b..00000000 --- a/bootstrap/k3s/api/v1beta1/k3sconfig.go +++ /dev/null @@ -1,212 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -// ServerConfiguration defines the desired state of k3s server configuration. -type ServerConfiguration struct { - // Database is the database configuration. - Database Database `json:"database,omitempty"` - - // Listener is the listener configuration. - Listener Listener `json:"listener,omitempty"` - - // Networking is the networking configuration. - Networking Networking `json:"networking,omitempty"` - - // KubernetesComponents is the kubernetes components configuration. - KubernetesComponents KubernetesComponents `json:"kubernetesComponents,omitempty"` - - // KubernetesProcesses is the kubernetes processes configuration. - KubernetesProcesses KubernetesProcesses `json:"kubernetesProcesses,omitempty"` - - // Agent is the agent configuration. - Agent AgentConfiguration `json:"agent,omitempty"` -} - -// AgentConfiguration defines the desired state of k3s agent configuration. -type AgentConfiguration struct { - // Node defines the k3s agent node configuration. - Node AgentNode `json:"node,omitempty"` - - // Runtime defines the k3s agent runtime configuration. - Runtime AgentRuntime `json:"runtime,omitempty"` - - // Networking defines the k3s agent networking configuration. - Networking AgentNetworking `json:"networking,omitempty"` - - // KubernetesAgentProcesses defines the k3s agent kubernetes processes configuration. - KubernetesAgentProcesses KubernetesAgentProcesses `json:"kubernetesAgentProcesses,omitempty"` -} - -// Database defines the desired state of k3s database configuration. -type Database struct { - // DataStoreEndPoint specify etcd, Mysql, Postgres, or Sqlite (default) data source name. - DataStoreEndPoint string `json:"dataStoreEndPoint,omitempty"` - - // DataStoreCAFile TLS Certificate Authority file used to secure datastore backend communication. - DataStoreCAFile string `json:"dataStoreCAFile,omitempty"` - - // DataStoreCertFile TLS certification file used to secure datastore backend communication. - DataStoreCertFile string `json:"dataStoreCertFile,omitempty"` - - // DataStoreKeyFile TLS key file used to secure datastore backend communication. - DataStoreKeyFile string `json:"dataStoreKeyFile,omitempty"` - - // ClusterInit initialize a new cluster using embedded Etcd. - ClusterInit *bool `json:"clusterInit,omitempty"` -} - -// Cluster is the desired state of k3s cluster configuration. -type Cluster struct { - // Token shared secret used to join a server or agent to a cluster. - Token string `json:"token,omitempty"` - - // TokenFile file containing the cluster-secret/token. - TokenFile string `json:"tokenFile,omitempty"` - - // Server which server to connect to, used to join a cluster. - Server string `json:"server,omitempty"` -} - -// Listener defines the desired state of k3s listener configuration. -type Listener struct { - // BindAddress k3s bind address. - BindAddress string `json:"bindAddress,omitempty"` - - // HTTPSListenPort HTTPS listen port. - HTTPSListenPort int `json:"httpsListenPort,omitempty"` - - // AdvertiseAddress IP address that apiserver uses to advertise to members of the cluster. - AdvertiseAddress string `json:"advertiseAddress,omitempty"` - - // AdvertisePort Port that apiserver uses to advertise to members of the cluster (default: listen-port). - AdvertisePort int `json:"advertisePort,omitempty"` - - // TLSSan Add additional hostname or IP as a Subject Alternative Name in the TLS cert. - TLSSan string `json:"tlsSan,omitempty"` -} - -// Networking defines the desired state of k3s networking configuration. -type Networking struct { - // ClusterCIDR Network CIDR to use for pod IPs. - ClusterCIDR string `json:"clusterCIDR,omitempty"` - - // ServiceCIDR Network CIDR to use for services IPs. - ServiceCIDR string `json:"serviceCIDR,omitempty"` - - // ServiceNodePortRange Port range to reserve for services with NodePort visibility. - ServiceNodePortRange string `json:"serviceNodePortRange,omitempty"` - - // ClusterDNS cluster IP for coredns service. Should be in your service-cidr range. - ClusterDNS string `json:"clusterDNS,omitempty"` - - // ClusterDomain cluster Domain. - ClusterDomain string `json:"clusterDomain,omitempty"` - - // FlannelBackend One of ‘none’, ‘vxlan’, ‘ipsec’, ‘host-gw’, or ‘wireguard’. (default: vxlan) - FlannelBackend string `json:"flannelBackend,omitempty"` -} - -// AgentNode defines the desired state of k3s agent node configuration. -type AgentNode struct { - // NodeName k3s node name. - NodeName string `json:"nodeName,omitempty"` - - // NodeLabels registering and starting kubelet with set of labels. - NodeLabels []string `json:"nodeLabels,omitempty"` - - // NodeTaints registering and starting kubelet with set of taints. - NodeTaints []string `json:"nodeTaints,omitempty"` - - // SeLinux Enable SELinux in containerd - SeLinux bool `json:"seLinux,omitempty"` - - // LBServerPort - // Local port for supervisor client load-balancer. - // If the supervisor and apiserver are not colocated an additional port 1 less than this port - // will also be used for the apiserver client load-balancer. (default: 6444) - LBServerPort int `json:"lbServerPort,omitempty"` - - // DataDir Folder to hold state. - DataDir string `json:"dataDir,omitempty"` -} - -// AgentRuntime defines the desired state of k3s agent runtime configuration. -type AgentRuntime struct { - // ContainerRuntimeEndpoint Disable embedded containerd and use alternative CRI implementation. - ContainerRuntimeEndpoint string `json:"containerRuntimeEndpoint,omitempty"` - - // PauseImage Customized pause image for containerd or Docker sandbox. - PauseImage string `json:"pauseImage,omitempty"` - - // PrivateRegistry Path to a private registry configuration file. - PrivateRegistry string `json:"privateRegistry,omitempty"` -} - -// AgentNetworking defines the desired state of k3s agent networking configuration. -type AgentNetworking struct { - // NodeIP IP address to advertise for node. - NodeIP string `json:"nodeIP,omitempty"` - - // NodeExternalIP External IP address to advertise for node. - NodeExternalIP string `json:"nodeExternalIP,omitempty"` - - // ResolvConf Path to Kubelet resolv.conf file. - ResolvConf string `json:"resolvConf,omitempty"` -} - -// KubernetesComponents defines the desired state of k3s kubernetes components configuration. -type KubernetesComponents struct { - // Disable do not deploy packaged components and delete any deployed components - // (valid items: coredns, servicelb, traefik,local-storage, metrics-server). - Disable string `json:"disable,omitempty"` - - // DisableKubeProxy disable running kube-proxy. - DisableKubeProxy bool `json:"disableKubeProxy,omitempty"` - - // DisableNetworkPolicy disable k3s default network policy controller. - DisableNetworkPolicy bool `json:"disableNetworkPolicy,omitempty"` - - // DisableHelmController disable Helm controller. - DisableHelmController bool `json:"disableHelmController,omitempty"` -} - -// KubernetesProcesses defines the desired state of kubernetes processes configuration. -type KubernetesProcesses struct { - // KubeAPIServerArgs is a customized flag for kube-apiserver process - // +optional - KubeAPIServerArgs []string `json:"kubeAPIServerArg,omitempty"` - - // KubeControllerManagerArgs is a customized flag for kube-controller-manager process - // +optional - KubeControllerManagerArgs []string `json:"kubeControllerManagerArgs,omitempty"` - - // KubeSchedulerArgs is a customized flag for kube-scheduler process - // +optional - KubeSchedulerArgs []string `json:"kubeSchedulerArgs,omitempty"` -} - -// KubernetesAgentProcesses defines the desired state of kubernetes agent processes configuration. -type KubernetesAgentProcesses struct { - // KubeletArgs Customized flag for kubelet process - // +optional - KubeletArgs []string `json:"kubeletArgs,omitempty"` - - // KubeProxyArgs Customized flag for kube-proxy process - // +optional - KubeProxyArgs []string `json:"kubeProxyArgs,omitempty"` -} diff --git a/bootstrap/k3s/api/v1beta1/k3sconfig_types.go b/bootstrap/k3s/api/v1beta1/k3sconfig_types.go deleted file mode 100644 index 572accb8..00000000 --- a/bootstrap/k3s/api/v1beta1/k3sconfig_types.go +++ /dev/null @@ -1,120 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" -) - -// K3sConfigSpec defines the desired state of K3sConfig -type K3sConfigSpec struct { - // Files specifies extra files to be passed to user_data upon creation. - // +optional - Files []bootstrapv1.File `json:"files,omitempty"` - - // Cluster defines the k3s cluster Options. - Cluster *Cluster `json:"cluster,omitempty"` - - // ServerConfiguration defines the k3s server configuration. - // +optional - ServerConfiguration *ServerConfiguration `json:"serverConfiguration,omitempty"` - - // AgentConfiguration defines the k3s agent configuration. - // +optional - AgentConfiguration *AgentConfiguration `json:"agentConfiguration,omitempty"` - - // PreK3sCommands specifies extra commands to run before k3s setup runs - // +optional - PreK3sCommands []string `json:"preK3sCommands,omitempty"` - - // PostK3sCommands specifies extra commands to run after k3s setup runs - // +optional - PostK3sCommands []string `json:"postK3sCommands,omitempty"` - - // Version specifies the k3s version - // +optional - Version string `json:"version,omitempty"` -} - -// K3sConfigStatus defines the observed state of K3sConfig -type K3sConfigStatus struct { - // Ready indicates the BootstrapData field is ready to be consumed - Ready bool `json:"ready,omitempty"` - - BootstrapData []byte `json:"bootstrapData,omitempty"` - - // DataSecretName is the name of the secret that stores the bootstrap data script. - // +optional - DataSecretName *string `json:"dataSecretName,omitempty"` - - // FailureReason will be set on non-retryable errors - // +optional - FailureReason string `json:"failureReason,omitempty"` - - // FailureMessage will be set on non-retryable errors - // +optional - FailureMessage string `json:"failureMessage,omitempty"` - - // ObservedGeneration is the latest generation observed by the controller. - // +optional - ObservedGeneration int64 `json:"observedGeneration,omitempty"` - - // Conditions defines current service state of the K3sConfig. - // +optional - Conditions clusterv1.Conditions `json:"conditions,omitempty"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:resource:path=k3sconfigs,scope=Namespaced,categories=cluster-api -// +kubebuilder:storageversion -// +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels['cluster\\.x-k8s\\.io/cluster-name']",description="Cluster" -// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of K3sConfig" - -// K3sConfig is the Schema for the k3sConfigs API -type K3sConfig struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec K3sConfigSpec `json:"spec,omitempty"` - Status K3sConfigStatus `json:"status,omitempty"` -} - -// GetConditions returns the set of conditions for this object. -func (c *K3sConfig) GetConditions() clusterv1.Conditions { - return c.Status.Conditions -} - -// SetConditions sets the conditions on this object. -func (c *K3sConfig) SetConditions(conditions clusterv1.Conditions) { - c.Status.Conditions = conditions -} - -//+kubebuilder:object:root=true - -// K3sConfigList contains a list of K3sConfig -type K3sConfigList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []K3sConfig `json:"items"` -} - -func init() { - SchemeBuilder.Register(&K3sConfig{}, &K3sConfigList{}) -} diff --git a/bootstrap/k3s/api/v1beta1/k3sconfig_webhook.go b/bootstrap/k3s/api/v1beta1/k3sconfig_webhook.go deleted file mode 100644 index 5d4b0948..00000000 --- a/bootstrap/k3s/api/v1beta1/k3sconfig_webhook.go +++ /dev/null @@ -1,146 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -var ( - conflictingFileSourceMsg = "only one of content or contentFrom may be specified for a single file" - missingSecretNameMsg = "secret file source must specify non-empty secret name" - missingSecretKeyMsg = "secret file source must specify non-empty secret key" - pathConflictMsg = "path property must be unique among all files" -) - -func (c *K3sConfig) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(c). - Complete() -} - -// +kubebuilder:webhook:verbs=create;update,path=/mutate-bootstrap-cluster-x-k8s-io-v1beta1-k3sconfig,mutating=true,failurePolicy=fail,sideEffects=None,groups=bootstrap.cluster.x-k8s.io,resources=k3sconfigs,versions=v1beta1,name=default.k3sconfig.bootstrap.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 - -var _ webhook.Defaulter = &K3sConfig{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (c *K3sConfig) Default() { - DefaultK3sConfigSpec(&c.Spec) -} - -// DefaultK3sConfigSpec defaults a K3sConfigSpec. -func DefaultK3sConfigSpec(c *K3sConfigSpec) { -} - -// +kubebuilder:webhook:verbs=create;update,path=/validate-bootstrap-cluster-x-k8s-io-v1beta1-k3sconfig,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=bootstrap.cluster.x-k8s.io,resources=k3sconfigs,versions=v1beta1,name=validation.k3sconfig.bootstrap.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 - -var _ webhook.Validator = &K3sConfig{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (c *K3sConfig) ValidateCreate() error { - return c.Spec.validate(c.Name) -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (c *K3sConfig) ValidateUpdate(old runtime.Object) error { - return c.Spec.validate(c.Name) -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (c *K3sConfig) ValidateDelete() error { - return nil -} - -func (c *K3sConfigSpec) validate(name string) error { - allErrs := c.Validate(field.NewPath("spec")) - - if len(allErrs) == 0 { - return nil - } - - return apierrors.NewInvalid(GroupVersion.WithKind("K3sConfig").GroupKind(), name, allErrs) -} - -// Validate ensures the K3sConfigSpec is valid. -func (c *K3sConfigSpec) Validate(pathPrefix *field.Path) field.ErrorList { - var allErrs field.ErrorList - - allErrs = append(allErrs, c.validateFiles(pathPrefix)...) - - return allErrs -} - -func (c *K3sConfigSpec) validateFiles(pathPrefix *field.Path) field.ErrorList { - var allErrs field.ErrorList - - knownPaths := map[string]struct{}{} - - for i := range c.Files { - file := c.Files[i] - if file.Content != "" && file.ContentFrom != nil { - allErrs = append( - allErrs, - field.Invalid( - pathPrefix.Child("files").Index(i), - file, - conflictingFileSourceMsg, - ), - ) - } - // n.b.: if we ever add types besides Secret as a ContentFrom - // Source, we must add webhook validation here for one of the - // sources being non-nil. - if file.ContentFrom != nil { - if file.ContentFrom.Secret.Name == "" { - allErrs = append( - allErrs, - field.Required( - pathPrefix.Child("files").Index(i).Child("contentFrom", "secret", "name"), - missingSecretNameMsg, - ), - ) - } - if file.ContentFrom.Secret.Key == "" { - allErrs = append( - allErrs, - field.Required( - pathPrefix.Child("files").Index(i).Child("contentFrom", "secret", "key"), - missingSecretKeyMsg, - ), - ) - } - } - _, conflict := knownPaths[file.Path] - if conflict { - allErrs = append( - allErrs, - field.Invalid( - pathPrefix.Child("files").Index(i).Child("path"), - file, - pathConflictMsg, - ), - ) - } - knownPaths[file.Path] = struct{}{} - } - - return allErrs -} diff --git a/bootstrap/k3s/api/v1beta1/k3sconfigtemplate_types.go b/bootstrap/k3s/api/v1beta1/k3sconfigtemplate_types.go deleted file mode 100644 index dce24f20..00000000 --- a/bootstrap/k3s/api/v1beta1/k3sconfigtemplate_types.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// K3sConfigTemplateSpec defines the desired state of K3sConfigTemplate -type K3sConfigTemplateSpec struct { - Template K3sConfigTemplateResource `json:"template"` -} - -// K3sConfigTemplateResource defines the Template structure -type K3sConfigTemplateResource struct { - Spec K3sConfigSpec `json:"spec,omitempty"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:resource:path=k3sconfigtemplates,scope=Namespaced,categories=cluster-api -// +kubebuilder:storageversion -// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of K3sConfigTemplate" - -// K3sConfigTemplate is the Schema for the k3sconfigtemplates API -type K3sConfigTemplate struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec K3sConfigTemplateSpec `json:"spec,omitempty"` -} - -//+kubebuilder:object:root=true - -// K3sConfigTemplateList contains a list of K3sConfigTemplate -type K3sConfigTemplateList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []K3sConfigTemplate `json:"items"` -} - -func init() { - SchemeBuilder.Register(&K3sConfigTemplate{}, &K3sConfigTemplateList{}) -} diff --git a/bootstrap/k3s/api/v1beta1/k3sconfigtemplate_webhook.go b/bootstrap/k3s/api/v1beta1/k3sconfigtemplate_webhook.go deleted file mode 100644 index 626334b4..00000000 --- a/bootstrap/k3s/api/v1beta1/k3sconfigtemplate_webhook.go +++ /dev/null @@ -1,71 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook" -) - -func (r *K3sConfigTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(r). - Complete() -} - -// +kubebuilder:webhook:verbs=create;update,path=/mutate-bootstrap-cluster-x-k8s-io-v1beta1-k3sconfigtemplate,mutating=true,failurePolicy=fail,groups=bootstrap.cluster.x-k8s.io,resources=k3sconfigtemplates,versions=v1beta1,name=default.k3sconfigtemplate.bootstrap.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 - -var _ webhook.Defaulter = &K3sConfigTemplate{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (r *K3sConfigTemplate) Default() { - DefaultK3sConfigSpec(&r.Spec.Template.Spec) -} - -// +kubebuilder:webhook:verbs=create;update,path=/validate-bootstrap-cluster-x-k8s-io-v1beta1-k3sconfigtemplate,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=bootstrap.cluster.x-k8s.io,resources=k3sconfigtemplates,versions=v1beta1,name=validation.k3sconfigtemplate.bootstrap.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 - -var _ webhook.Validator = &K3sConfigTemplate{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (r *K3sConfigTemplate) ValidateCreate() error { - return r.Spec.validate(r.Name) -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (r *K3sConfigTemplate) ValidateUpdate(old runtime.Object) error { - return r.Spec.validate(r.Name) -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (r *K3sConfigTemplate) ValidateDelete() error { - return nil -} - -func (r *K3sConfigTemplateSpec) validate(name string) error { - var allErrs field.ErrorList - - allErrs = append(allErrs, r.Template.Spec.Validate(field.NewPath("spec", "template", "spec"))...) - - if len(allErrs) == 0 { - return nil - } - - return apierrors.NewInvalid(GroupVersion.WithKind("K3sConfigTemplate").GroupKind(), name, allErrs) -} diff --git a/bootstrap/k3s/api/v1beta1/zz_generated.deepcopy.go b/bootstrap/k3s/api/v1beta1/zz_generated.deepcopy.go deleted file mode 100644 index ab819fe2..00000000 --- a/bootstrap/k3s/api/v1beta1/zz_generated.deepcopy.go +++ /dev/null @@ -1,486 +0,0 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by controller-gen. DO NOT EDIT. - -package v1beta1 - -import ( - "k8s.io/apimachinery/pkg/runtime" - cluster_apiapiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" - apiv1beta1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AgentConfiguration) DeepCopyInto(out *AgentConfiguration) { - *out = *in - in.Node.DeepCopyInto(&out.Node) - out.Runtime = in.Runtime - out.Networking = in.Networking - in.KubernetesAgentProcesses.DeepCopyInto(&out.KubernetesAgentProcesses) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentConfiguration. -func (in *AgentConfiguration) DeepCopy() *AgentConfiguration { - if in == nil { - return nil - } - out := new(AgentConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AgentNetworking) DeepCopyInto(out *AgentNetworking) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentNetworking. -func (in *AgentNetworking) DeepCopy() *AgentNetworking { - if in == nil { - return nil - } - out := new(AgentNetworking) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AgentNode) DeepCopyInto(out *AgentNode) { - *out = *in - if in.NodeLabels != nil { - in, out := &in.NodeLabels, &out.NodeLabels - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.NodeTaints != nil { - in, out := &in.NodeTaints, &out.NodeTaints - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentNode. -func (in *AgentNode) DeepCopy() *AgentNode { - if in == nil { - return nil - } - out := new(AgentNode) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AgentRuntime) DeepCopyInto(out *AgentRuntime) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentRuntime. -func (in *AgentRuntime) DeepCopy() *AgentRuntime { - if in == nil { - return nil - } - out := new(AgentRuntime) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Cluster) DeepCopyInto(out *Cluster) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cluster. -func (in *Cluster) DeepCopy() *Cluster { - if in == nil { - return nil - } - out := new(Cluster) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Database) DeepCopyInto(out *Database) { - *out = *in - if in.ClusterInit != nil { - in, out := &in.ClusterInit, &out.ClusterInit - *out = new(bool) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Database. -func (in *Database) DeepCopy() *Database { - if in == nil { - return nil - } - out := new(Database) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sConfig) DeepCopyInto(out *K3sConfig) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sConfig. -func (in *K3sConfig) DeepCopy() *K3sConfig { - if in == nil { - return nil - } - out := new(K3sConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *K3sConfig) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sConfigList) DeepCopyInto(out *K3sConfigList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]K3sConfig, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sConfigList. -func (in *K3sConfigList) DeepCopy() *K3sConfigList { - if in == nil { - return nil - } - out := new(K3sConfigList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *K3sConfigList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sConfigSpec) DeepCopyInto(out *K3sConfigSpec) { - *out = *in - if in.Files != nil { - in, out := &in.Files, &out.Files - *out = make([]apiv1beta1.File, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Cluster != nil { - in, out := &in.Cluster, &out.Cluster - *out = new(Cluster) - **out = **in - } - if in.ServerConfiguration != nil { - in, out := &in.ServerConfiguration, &out.ServerConfiguration - *out = new(ServerConfiguration) - (*in).DeepCopyInto(*out) - } - if in.AgentConfiguration != nil { - in, out := &in.AgentConfiguration, &out.AgentConfiguration - *out = new(AgentConfiguration) - (*in).DeepCopyInto(*out) - } - if in.PreK3sCommands != nil { - in, out := &in.PreK3sCommands, &out.PreK3sCommands - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.PostK3sCommands != nil { - in, out := &in.PostK3sCommands, &out.PostK3sCommands - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sConfigSpec. -func (in *K3sConfigSpec) DeepCopy() *K3sConfigSpec { - if in == nil { - return nil - } - out := new(K3sConfigSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sConfigStatus) DeepCopyInto(out *K3sConfigStatus) { - *out = *in - if in.BootstrapData != nil { - in, out := &in.BootstrapData, &out.BootstrapData - *out = make([]byte, len(*in)) - copy(*out, *in) - } - if in.DataSecretName != nil { - in, out := &in.DataSecretName, &out.DataSecretName - *out = new(string) - **out = **in - } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make(cluster_apiapiv1beta1.Conditions, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sConfigStatus. -func (in *K3sConfigStatus) DeepCopy() *K3sConfigStatus { - if in == nil { - return nil - } - out := new(K3sConfigStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sConfigTemplate) DeepCopyInto(out *K3sConfigTemplate) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sConfigTemplate. -func (in *K3sConfigTemplate) DeepCopy() *K3sConfigTemplate { - if in == nil { - return nil - } - out := new(K3sConfigTemplate) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *K3sConfigTemplate) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sConfigTemplateList) DeepCopyInto(out *K3sConfigTemplateList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]K3sConfigTemplate, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sConfigTemplateList. -func (in *K3sConfigTemplateList) DeepCopy() *K3sConfigTemplateList { - if in == nil { - return nil - } - out := new(K3sConfigTemplateList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *K3sConfigTemplateList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sConfigTemplateResource) DeepCopyInto(out *K3sConfigTemplateResource) { - *out = *in - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sConfigTemplateResource. -func (in *K3sConfigTemplateResource) DeepCopy() *K3sConfigTemplateResource { - if in == nil { - return nil - } - out := new(K3sConfigTemplateResource) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sConfigTemplateSpec) DeepCopyInto(out *K3sConfigTemplateSpec) { - *out = *in - in.Template.DeepCopyInto(&out.Template) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sConfigTemplateSpec. -func (in *K3sConfigTemplateSpec) DeepCopy() *K3sConfigTemplateSpec { - if in == nil { - return nil - } - out := new(K3sConfigTemplateSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KubernetesAgentProcesses) DeepCopyInto(out *KubernetesAgentProcesses) { - *out = *in - if in.KubeletArgs != nil { - in, out := &in.KubeletArgs, &out.KubeletArgs - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.KubeProxyArgs != nil { - in, out := &in.KubeProxyArgs, &out.KubeProxyArgs - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesAgentProcesses. -func (in *KubernetesAgentProcesses) DeepCopy() *KubernetesAgentProcesses { - if in == nil { - return nil - } - out := new(KubernetesAgentProcesses) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KubernetesComponents) DeepCopyInto(out *KubernetesComponents) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesComponents. -func (in *KubernetesComponents) DeepCopy() *KubernetesComponents { - if in == nil { - return nil - } - out := new(KubernetesComponents) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KubernetesProcesses) DeepCopyInto(out *KubernetesProcesses) { - *out = *in - if in.KubeAPIServerArgs != nil { - in, out := &in.KubeAPIServerArgs, &out.KubeAPIServerArgs - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.KubeControllerManagerArgs != nil { - in, out := &in.KubeControllerManagerArgs, &out.KubeControllerManagerArgs - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.KubeSchedulerArgs != nil { - in, out := &in.KubeSchedulerArgs, &out.KubeSchedulerArgs - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesProcesses. -func (in *KubernetesProcesses) DeepCopy() *KubernetesProcesses { - if in == nil { - return nil - } - out := new(KubernetesProcesses) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Listener) DeepCopyInto(out *Listener) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Listener. -func (in *Listener) DeepCopy() *Listener { - if in == nil { - return nil - } - out := new(Listener) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Networking) DeepCopyInto(out *Networking) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Networking. -func (in *Networking) DeepCopy() *Networking { - if in == nil { - return nil - } - out := new(Networking) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ServerConfiguration) DeepCopyInto(out *ServerConfiguration) { - *out = *in - in.Database.DeepCopyInto(&out.Database) - out.Listener = in.Listener - out.Networking = in.Networking - out.KubernetesComponents = in.KubernetesComponents - in.KubernetesProcesses.DeepCopyInto(&out.KubernetesProcesses) - in.Agent.DeepCopyInto(&out.Agent) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServerConfiguration. -func (in *ServerConfiguration) DeepCopy() *ServerConfiguration { - if in == nil { - return nil - } - out := new(ServerConfiguration) - in.DeepCopyInto(out) - return out -} diff --git a/bootstrap/k3s/config/certmanager/certificate.yaml b/bootstrap/k3s/config/certmanager/certificate.yaml deleted file mode 100644 index 0f645290..00000000 --- a/bootstrap/k3s/config/certmanager/certificate.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for breaking changes -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize - dnsNames: - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: $(SERVICE_NAME)-cert # this secret will not be prefixed, since it's not managed by kustomize \ No newline at end of file diff --git a/bootstrap/k3s/config/certmanager/kustomization.yaml b/bootstrap/k3s/config/certmanager/kustomization.yaml deleted file mode 100644 index bebea5a5..00000000 --- a/bootstrap/k3s/config/certmanager/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -resources: -- certificate.yaml - -configurations: -- kustomizeconfig.yaml diff --git a/bootstrap/k3s/config/certmanager/kustomizeconfig.yaml b/bootstrap/k3s/config/certmanager/kustomizeconfig.yaml deleted file mode 100644 index 28a895a4..00000000 --- a/bootstrap/k3s/config/certmanager/kustomizeconfig.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# This configuration is for teaching kustomize how to update name ref and var substitution -nameReference: -- kind: Issuer - group: cert-manager.io - fieldSpecs: - - kind: Certificate - group: cert-manager.io - path: spec/issuerRef/name - -varReference: -- kind: Certificate - group: cert-manager.io - path: spec/commonName -- kind: Certificate - group: cert-manager.io - path: spec/dnsNames -- kind: Certificate - group: cert-manager.io - path: spec/secretName diff --git a/bootstrap/k3s/config/crd/bases/bootstrap.cluster.x-k8s.io_k3sconfigs.yaml b/bootstrap/k3s/config/crd/bases/bootstrap.cluster.x-k8s.io_k3sconfigs.yaml deleted file mode 100644 index bc58504e..00000000 --- a/bootstrap/k3s/config/crd/bases/bootstrap.cluster.x-k8s.io_k3sconfigs.yaml +++ /dev/null @@ -1,496 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.1 - creationTimestamp: null - name: k3sconfigs.bootstrap.cluster.x-k8s.io -spec: - group: bootstrap.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: K3sConfig - listKind: K3sConfigList - plural: k3sconfigs - singular: k3sconfig - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Cluster - jsonPath: .metadata.labels['cluster\.x-k8s\.io/cluster-name'] - name: Cluster - type: string - - description: Time duration since creation of K3sConfig - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: K3sConfig is the Schema for the k3sConfigs API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: K3sConfigSpec defines the desired state of K3sConfig - properties: - agentConfiguration: - description: AgentConfiguration defines the k3s agent configuration. - properties: - kubernetesAgentProcesses: - description: KubernetesAgentProcesses defines the k3s agent kubernetes - processes configuration. - properties: - kubeProxyArgs: - description: KubeProxyArgs Customized flag for kube-proxy - process - items: - type: string - type: array - kubeletArgs: - description: KubeletArgs Customized flag for kubelet process - items: - type: string - type: array - type: object - networking: - description: Networking defines the k3s agent networking configuration. - properties: - nodeExternalIP: - description: NodeExternalIP External IP address to advertise - for node. - type: string - nodeIP: - description: NodeIP IP address to advertise for node. - type: string - resolvConf: - description: ResolvConf Path to Kubelet resolv.conf file. - type: string - type: object - node: - description: Node defines the k3s agent node configuration. - properties: - dataDir: - description: DataDir Folder to hold state. - type: string - lbServerPort: - description: 'LBServerPort Local port for supervisor client - load-balancer. If the supervisor and apiserver are not colocated - an additional port 1 less than this port will also be used - for the apiserver client load-balancer. (default: 6444)' - type: integer - nodeLabels: - description: NodeLabels registering and starting kubelet with - set of labels. - items: - type: string - type: array - nodeName: - description: NodeName k3s node name. - type: string - nodeTaints: - description: NodeTaints registering and starting kubelet with - set of taints. - items: - type: string - type: array - seLinux: - description: SeLinux Enable SELinux in containerd - type: boolean - type: object - runtime: - description: Runtime defines the k3s agent runtime configuration. - properties: - containerRuntimeEndpoint: - description: ContainerRuntimeEndpoint Disable embedded containerd - and use alternative CRI implementation. - type: string - pauseImage: - description: PauseImage Customized pause image for containerd - or Docker sandbox. - type: string - privateRegistry: - description: PrivateRegistry Path to a private registry configuration - file. - type: string - type: object - type: object - cluster: - description: Cluster defines the k3s cluster Options. - properties: - server: - description: Server which server to connect to, used to join a - cluster. - type: string - token: - description: Token shared secret used to join a server or agent - to a cluster. - type: string - tokenFile: - description: TokenFile file containing the cluster-secret/token. - type: string - type: object - files: - description: Files specifies extra files to be passed to user_data - upon creation. - items: - description: File defines the input for generating write_files in - cloud-init. - properties: - append: - description: Append specifies whether to append Content to existing - file if Path exists. - type: boolean - content: - description: Content is the actual content of the file. - type: string - contentFrom: - description: ContentFrom is a referenced source of content to - populate the file. - properties: - secret: - description: Secret represents a secret that should populate - this file. - properties: - key: - description: Key is the key in the secret's data map - for this value. - type: string - name: - description: Name of the secret in the KubeadmBootstrapConfig's - namespace to use. - type: string - required: - - key - - name - type: object - required: - - secret - type: object - encoding: - description: Encoding specifies the encoding of the file contents. - enum: - - base64 - - gzip - - gzip+base64 - type: string - owner: - description: Owner specifies the ownership of the file, e.g. - "root:root". - type: string - path: - description: Path specifies the full path on disk where to store - the file. - type: string - permissions: - description: Permissions specifies the permissions to assign - to the file, e.g. "0640". - type: string - required: - - path - type: object - type: array - postK3sCommands: - description: PostK3sCommands specifies extra commands to run after - k3s setup runs - items: - type: string - type: array - preK3sCommands: - description: PreK3sCommands specifies extra commands to run before - k3s setup runs - items: - type: string - type: array - serverConfiguration: - description: ServerConfiguration defines the k3s server configuration. - properties: - agent: - description: Agent is the agent configuration. - properties: - kubernetesAgentProcesses: - description: KubernetesAgentProcesses defines the k3s agent - kubernetes processes configuration. - properties: - kubeProxyArgs: - description: KubeProxyArgs Customized flag for kube-proxy - process - items: - type: string - type: array - kubeletArgs: - description: KubeletArgs Customized flag for kubelet process - items: - type: string - type: array - type: object - networking: - description: Networking defines the k3s agent networking configuration. - properties: - nodeExternalIP: - description: NodeExternalIP External IP address to advertise - for node. - type: string - nodeIP: - description: NodeIP IP address to advertise for node. - type: string - resolvConf: - description: ResolvConf Path to Kubelet resolv.conf file. - type: string - type: object - node: - description: Node defines the k3s agent node configuration. - properties: - dataDir: - description: DataDir Folder to hold state. - type: string - lbServerPort: - description: 'LBServerPort Local port for supervisor client - load-balancer. If the supervisor and apiserver are not - colocated an additional port 1 less than this port will - also be used for the apiserver client load-balancer. - (default: 6444)' - type: integer - nodeLabels: - description: NodeLabels registering and starting kubelet - with set of labels. - items: - type: string - type: array - nodeName: - description: NodeName k3s node name. - type: string - nodeTaints: - description: NodeTaints registering and starting kubelet - with set of taints. - items: - type: string - type: array - seLinux: - description: SeLinux Enable SELinux in containerd - type: boolean - type: object - runtime: - description: Runtime defines the k3s agent runtime configuration. - properties: - containerRuntimeEndpoint: - description: ContainerRuntimeEndpoint Disable embedded - containerd and use alternative CRI implementation. - type: string - pauseImage: - description: PauseImage Customized pause image for containerd - or Docker sandbox. - type: string - privateRegistry: - description: PrivateRegistry Path to a private registry - configuration file. - type: string - type: object - type: object - database: - description: Database is the database configuration. - properties: - clusterInit: - description: ClusterInit initialize a new cluster using embedded - Etcd. - type: boolean - dataStoreCAFile: - description: DataStoreCAFile TLS Certificate Authority file - used to secure datastore backend communication. - type: string - dataStoreCertFile: - description: DataStoreCertFile TLS certification file used - to secure datastore backend communication. - type: string - dataStoreEndPoint: - description: DataStoreEndPoint specify etcd, Mysql, Postgres, - or Sqlite (default) data source name. - type: string - dataStoreKeyFile: - description: DataStoreKeyFile TLS key file used to secure - datastore backend communication. - type: string - type: object - kubernetesComponents: - description: KubernetesComponents is the kubernetes components - configuration. - properties: - disable: - description: 'Disable do not deploy packaged components and - delete any deployed components (valid items: coredns, servicelb, - traefik,local-storage, metrics-server).' - type: string - disableHelmController: - description: DisableHelmController disable Helm controller. - type: boolean - disableKubeProxy: - description: DisableKubeProxy disable running kube-proxy. - type: boolean - disableNetworkPolicy: - description: DisableNetworkPolicy disable k3s default network - policy controller. - type: boolean - type: object - kubernetesProcesses: - description: KubernetesProcesses is the kubernetes processes configuration. - properties: - kubeAPIServerArg: - description: KubeAPIServerArgs is a customized flag for kube-apiserver - process - items: - type: string - type: array - kubeControllerManagerArgs: - description: KubeControllerManagerArgs is a customized flag - for kube-controller-manager process - items: - type: string - type: array - kubeSchedulerArgs: - description: KubeSchedulerArgs is a customized flag for kube-scheduler - process - items: - type: string - type: array - type: object - listener: - description: Listener is the listener configuration. - properties: - advertiseAddress: - description: AdvertiseAddress IP address that apiserver uses - to advertise to members of the cluster. - type: string - advertisePort: - description: 'AdvertisePort Port that apiserver uses to advertise - to members of the cluster (default: listen-port).' - type: integer - bindAddress: - description: BindAddress k3s bind address. - type: string - httpsListenPort: - description: HTTPSListenPort HTTPS listen port. - type: integer - tlsSan: - description: TLSSan Add additional hostname or IP as a Subject - Alternative Name in the TLS cert. - type: string - type: object - networking: - description: Networking is the networking configuration. - properties: - clusterCIDR: - description: ClusterCIDR Network CIDR to use for pod IPs. - type: string - clusterDNS: - description: ClusterDNS cluster IP for coredns service. Should - be in your service-cidr range. - type: string - clusterDomain: - description: ClusterDomain cluster Domain. - type: string - flannelBackend: - description: 'FlannelBackend One of ‘none’, ‘vxlan’, ‘ipsec’, - ‘host-gw’, or ‘wireguard’. (default: vxlan)' - type: string - serviceCIDR: - description: ServiceCIDR Network CIDR to use for services - IPs. - type: string - serviceNodePortRange: - description: ServiceNodePortRange Port range to reserve for - services with NodePort visibility. - type: string - type: object - type: object - version: - description: Version specifies the k3s version - type: string - type: object - status: - description: K3sConfigStatus defines the observed state of K3sConfig - properties: - bootstrapData: - format: byte - type: string - conditions: - description: Conditions defines current service state of the K3sConfig. - items: - description: Condition defines an observation of a Cluster API resource - operational state. - properties: - lastTransitionTime: - description: Last time the condition transitioned from one status - to another. This should be when the underlying condition changed. - If that is not known, then using the time when the API field - changed is acceptable. - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. This field may be empty. - type: string - reason: - description: The reason for the condition's last transition - in CamelCase. The specific API may choose whether or not this - field is considered a guaranteed API. This field may not be - empty. - type: string - severity: - description: Severity provides an explicit classification of - Reason code, so the users or machines can immediately understand - the current situation and act accordingly. The Severity field - MUST be set only when Status=False. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of condition in CamelCase or in foo.example.com/CamelCase. - Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. - type: string - required: - - lastTransitionTime - - status - - type - type: object - type: array - dataSecretName: - description: DataSecretName is the name of the secret that stores - the bootstrap data script. - type: string - failureMessage: - description: FailureMessage will be set on non-retryable errors - type: string - failureReason: - description: FailureReason will be set on non-retryable errors - type: string - observedGeneration: - description: ObservedGeneration is the latest generation observed - by the controller. - format: int64 - type: integer - ready: - description: Ready indicates the BootstrapData field is ready to be - consumed - type: boolean - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/bootstrap/k3s/config/crd/bases/bootstrap.cluster.x-k8s.io_k3sconfigtemplates.yaml b/bootstrap/k3s/config/crd/bases/bootstrap.cluster.x-k8s.io_k3sconfigtemplates.yaml deleted file mode 100644 index 9dce789b..00000000 --- a/bootstrap/k3s/config/crd/bases/bootstrap.cluster.x-k8s.io_k3sconfigtemplates.yaml +++ /dev/null @@ -1,445 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.1 - creationTimestamp: null - name: k3sconfigtemplates.bootstrap.cluster.x-k8s.io -spec: - group: bootstrap.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: K3sConfigTemplate - listKind: K3sConfigTemplateList - plural: k3sconfigtemplates - singular: k3sconfigtemplate - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Time duration since creation of K3sConfigTemplate - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: K3sConfigTemplate is the Schema for the k3sconfigtemplates API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: K3sConfigTemplateSpec defines the desired state of K3sConfigTemplate - properties: - template: - description: K3sConfigTemplateResource defines the Template structure - properties: - spec: - description: K3sConfigSpec defines the desired state of K3sConfig - properties: - agentConfiguration: - description: AgentConfiguration defines the k3s agent configuration. - properties: - kubernetesAgentProcesses: - description: KubernetesAgentProcesses defines the k3s - agent kubernetes processes configuration. - properties: - kubeProxyArgs: - description: KubeProxyArgs Customized flag for kube-proxy - process - items: - type: string - type: array - kubeletArgs: - description: KubeletArgs Customized flag for kubelet - process - items: - type: string - type: array - type: object - networking: - description: Networking defines the k3s agent networking - configuration. - properties: - nodeExternalIP: - description: NodeExternalIP External IP address to - advertise for node. - type: string - nodeIP: - description: NodeIP IP address to advertise for node. - type: string - resolvConf: - description: ResolvConf Path to Kubelet resolv.conf - file. - type: string - type: object - node: - description: Node defines the k3s agent node configuration. - properties: - dataDir: - description: DataDir Folder to hold state. - type: string - lbServerPort: - description: 'LBServerPort Local port for supervisor - client load-balancer. If the supervisor and apiserver - are not colocated an additional port 1 less than - this port will also be used for the apiserver client - load-balancer. (default: 6444)' - type: integer - nodeLabels: - description: NodeLabels registering and starting kubelet - with set of labels. - items: - type: string - type: array - nodeName: - description: NodeName k3s node name. - type: string - nodeTaints: - description: NodeTaints registering and starting kubelet - with set of taints. - items: - type: string - type: array - seLinux: - description: SeLinux Enable SELinux in containerd - type: boolean - type: object - runtime: - description: Runtime defines the k3s agent runtime configuration. - properties: - containerRuntimeEndpoint: - description: ContainerRuntimeEndpoint Disable embedded - containerd and use alternative CRI implementation. - type: string - pauseImage: - description: PauseImage Customized pause image for - containerd or Docker sandbox. - type: string - privateRegistry: - description: PrivateRegistry Path to a private registry - configuration file. - type: string - type: object - type: object - cluster: - description: Cluster defines the k3s cluster Options. - properties: - server: - description: Server which server to connect to, used to - join a cluster. - type: string - token: - description: Token shared secret used to join a server - or agent to a cluster. - type: string - tokenFile: - description: TokenFile file containing the cluster-secret/token. - type: string - type: object - files: - description: Files specifies extra files to be passed to user_data - upon creation. - items: - description: File defines the input for generating write_files - in cloud-init. - properties: - append: - description: Append specifies whether to append Content - to existing file if Path exists. - type: boolean - content: - description: Content is the actual content of the file. - type: string - contentFrom: - description: ContentFrom is a referenced source of content - to populate the file. - properties: - secret: - description: Secret represents a secret that should - populate this file. - properties: - key: - description: Key is the key in the secret's - data map for this value. - type: string - name: - description: Name of the secret in the KubeadmBootstrapConfig's - namespace to use. - type: string - required: - - key - - name - type: object - required: - - secret - type: object - encoding: - description: Encoding specifies the encoding of the - file contents. - enum: - - base64 - - gzip - - gzip+base64 - type: string - owner: - description: Owner specifies the ownership of the file, - e.g. "root:root". - type: string - path: - description: Path specifies the full path on disk where - to store the file. - type: string - permissions: - description: Permissions specifies the permissions to - assign to the file, e.g. "0640". - type: string - required: - - path - type: object - type: array - postK3sCommands: - description: PostK3sCommands specifies extra commands to run - after k3s setup runs - items: - type: string - type: array - preK3sCommands: - description: PreK3sCommands specifies extra commands to run - before k3s setup runs - items: - type: string - type: array - serverConfiguration: - description: ServerConfiguration defines the k3s server configuration. - properties: - agent: - description: Agent is the agent configuration. - properties: - kubernetesAgentProcesses: - description: KubernetesAgentProcesses defines the - k3s agent kubernetes processes configuration. - properties: - kubeProxyArgs: - description: KubeProxyArgs Customized flag for - kube-proxy process - items: - type: string - type: array - kubeletArgs: - description: KubeletArgs Customized flag for kubelet - process - items: - type: string - type: array - type: object - networking: - description: Networking defines the k3s agent networking - configuration. - properties: - nodeExternalIP: - description: NodeExternalIP External IP address - to advertise for node. - type: string - nodeIP: - description: NodeIP IP address to advertise for - node. - type: string - resolvConf: - description: ResolvConf Path to Kubelet resolv.conf - file. - type: string - type: object - node: - description: Node defines the k3s agent node configuration. - properties: - dataDir: - description: DataDir Folder to hold state. - type: string - lbServerPort: - description: 'LBServerPort Local port for supervisor - client load-balancer. If the supervisor and - apiserver are not colocated an additional port - 1 less than this port will also be used for - the apiserver client load-balancer. (default: - 6444)' - type: integer - nodeLabels: - description: NodeLabels registering and starting - kubelet with set of labels. - items: - type: string - type: array - nodeName: - description: NodeName k3s node name. - type: string - nodeTaints: - description: NodeTaints registering and starting - kubelet with set of taints. - items: - type: string - type: array - seLinux: - description: SeLinux Enable SELinux in containerd - type: boolean - type: object - runtime: - description: Runtime defines the k3s agent runtime - configuration. - properties: - containerRuntimeEndpoint: - description: ContainerRuntimeEndpoint Disable - embedded containerd and use alternative CRI - implementation. - type: string - pauseImage: - description: PauseImage Customized pause image - for containerd or Docker sandbox. - type: string - privateRegistry: - description: PrivateRegistry Path to a private - registry configuration file. - type: string - type: object - type: object - database: - description: Database is the database configuration. - properties: - clusterInit: - description: ClusterInit initialize a new cluster - using embedded Etcd. - type: boolean - dataStoreCAFile: - description: DataStoreCAFile TLS Certificate Authority - file used to secure datastore backend communication. - type: string - dataStoreCertFile: - description: DataStoreCertFile TLS certification file - used to secure datastore backend communication. - type: string - dataStoreEndPoint: - description: DataStoreEndPoint specify etcd, Mysql, - Postgres, or Sqlite (default) data source name. - type: string - dataStoreKeyFile: - description: DataStoreKeyFile TLS key file used to - secure datastore backend communication. - type: string - type: object - kubernetesComponents: - description: KubernetesComponents is the kubernetes components - configuration. - properties: - disable: - description: 'Disable do not deploy packaged components - and delete any deployed components (valid items: - coredns, servicelb, traefik,local-storage, metrics-server).' - type: string - disableHelmController: - description: DisableHelmController disable Helm controller. - type: boolean - disableKubeProxy: - description: DisableKubeProxy disable running kube-proxy. - type: boolean - disableNetworkPolicy: - description: DisableNetworkPolicy disable k3s default - network policy controller. - type: boolean - type: object - kubernetesProcesses: - description: KubernetesProcesses is the kubernetes processes - configuration. - properties: - kubeAPIServerArg: - description: KubeAPIServerArgs is a customized flag - for kube-apiserver process - items: - type: string - type: array - kubeControllerManagerArgs: - description: KubeControllerManagerArgs is a customized - flag for kube-controller-manager process - items: - type: string - type: array - kubeSchedulerArgs: - description: KubeSchedulerArgs is a customized flag - for kube-scheduler process - items: - type: string - type: array - type: object - listener: - description: Listener is the listener configuration. - properties: - advertiseAddress: - description: AdvertiseAddress IP address that apiserver - uses to advertise to members of the cluster. - type: string - advertisePort: - description: 'AdvertisePort Port that apiserver uses - to advertise to members of the cluster (default: - listen-port).' - type: integer - bindAddress: - description: BindAddress k3s bind address. - type: string - httpsListenPort: - description: HTTPSListenPort HTTPS listen port. - type: integer - tlsSan: - description: TLSSan Add additional hostname or IP - as a Subject Alternative Name in the TLS cert. - type: string - type: object - networking: - description: Networking is the networking configuration. - properties: - clusterCIDR: - description: ClusterCIDR Network CIDR to use for pod - IPs. - type: string - clusterDNS: - description: ClusterDNS cluster IP for coredns service. - Should be in your service-cidr range. - type: string - clusterDomain: - description: ClusterDomain cluster Domain. - type: string - flannelBackend: - description: 'FlannelBackend One of ‘none’, ‘vxlan’, - ‘ipsec’, ‘host-gw’, or ‘wireguard’. (default: vxlan)' - type: string - serviceCIDR: - description: ServiceCIDR Network CIDR to use for services - IPs. - type: string - serviceNodePortRange: - description: ServiceNodePortRange Port range to reserve - for services with NodePort visibility. - type: string - type: object - type: object - version: - description: Version specifies the k3s version - type: string - type: object - type: object - required: - - template - type: object - type: object - served: true - storage: true - subresources: {} diff --git a/bootstrap/k3s/config/crd/kustomization.yaml b/bootstrap/k3s/config/crd/kustomization.yaml deleted file mode 100644 index 71b422e9..00000000 --- a/bootstrap/k3s/config/crd/kustomization.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# This kustomization.yaml is not intended to be run by itself, -# since it depends on service name and namespace that are out of this kustomize package. -# It should be run by config/default -resources: -- bases/bootstrap.cluster.x-k8s.io_k3sconfigs.yaml -- bases/bootstrap.cluster.x-k8s.io_k3sconfigtemplates.yaml -#+kubebuilder:scaffold:crdkustomizeresource - -commonLabels: - cluster.x-k8s.io/v1beta1: v1beta1 - -patchesStrategicMerge: -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. -# patches here are for enabling the conversion webhook for each CRD -- patches/webhook_in_k3sconfigs.yaml -- patches/webhook_in_k3sconfigtemplates.yaml -#+kubebuilder:scaffold:crdkustomizewebhookpatch - -# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. -# patches here are for enabling the CA injection for each CRD -- patches/cainjection_in_k3sconfigs.yaml -- patches/cainjection_in_k3sconfigtemplates.yaml -#+kubebuilder:scaffold:crdkustomizecainjectionpatch - -# the following config is for teaching kustomize how to do kustomization for CRDs. -configurations: -- kustomizeconfig.yaml diff --git a/bootstrap/k3s/config/crd/kustomizeconfig.yaml b/bootstrap/k3s/config/crd/kustomizeconfig.yaml deleted file mode 100644 index ec5c150a..00000000 --- a/bootstrap/k3s/config/crd/kustomizeconfig.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# This file is for teaching kustomize how to substitute name and namespace reference in CRD -nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/name - -namespace: -- kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/namespace - create: false - -varReference: -- path: metadata/annotations diff --git a/bootstrap/k3s/config/crd/patches/cainjection_in_k3sconfigs.yaml b/bootstrap/k3s/config/crd/patches/cainjection_in_k3sconfigs.yaml deleted file mode 100644 index 89973785..00000000 --- a/bootstrap/k3s/config/crd/patches/cainjection_in_k3sconfigs.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: k3sconfigs.bootstrap.cluster.x-k8s.io diff --git a/bootstrap/k3s/config/crd/patches/cainjection_in_k3sconfigtemplates.yaml b/bootstrap/k3s/config/crd/patches/cainjection_in_k3sconfigtemplates.yaml deleted file mode 100644 index 5722b7b3..00000000 --- a/bootstrap/k3s/config/crd/patches/cainjection_in_k3sconfigtemplates.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: k3sconfigtemplates.bootstrap.cluster.x-k8s.io diff --git a/bootstrap/k3s/config/crd/patches/webhook_in_k3sconfigs.yaml b/bootstrap/k3s/config/crd/patches/webhook_in_k3sconfigs.yaml deleted file mode 100644 index 75b66bfe..00000000 --- a/bootstrap/k3s/config/crd/patches/webhook_in_k3sconfigs.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: k3sconfigs.bootstrap.cluster.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: ["v1", "v1beta1"] - clientConfig: - # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, - # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) - caBundle: Cg== - service: - namespace: system - name: webhook-service - path: /convert diff --git a/bootstrap/k3s/config/crd/patches/webhook_in_k3sconfigtemplates.yaml b/bootstrap/k3s/config/crd/patches/webhook_in_k3sconfigtemplates.yaml deleted file mode 100644 index 79e1942f..00000000 --- a/bootstrap/k3s/config/crd/patches/webhook_in_k3sconfigtemplates.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: k3sconfigtemplates.bootstrap.cluster.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: ["v1", "v1beta1"] - clientConfig: - # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, - # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) - caBundle: Cg== - service: - namespace: system - name: webhook-service - path: /convert diff --git a/bootstrap/k3s/config/default/kustomization.yaml b/bootstrap/k3s/config/default/kustomization.yaml deleted file mode 100644 index 105a8e5f..00000000 --- a/bootstrap/k3s/config/default/kustomization.yaml +++ /dev/null @@ -1,54 +0,0 @@ -namePrefix: capkk-k3s-bootstrap- -namespace: capkk-k3s-bootstrap-system - -commonLabels: - cluster.x-k8s.io/provider: "bootstrap-k3s" - -resources: - - namespace.yaml - -bases: - - ../rbac - - ../manager - - ../crd - - ../certmanager - - ../webhook - -patchesStrategicMerge: - # Provide customizable hook for make targets. - - manager_image_patch.yaml - - manager_pull_policy.yaml - # Enable webhook. - - manager_webhook_patch.yaml - # Inject certificate in the webhook definition. - - webhookcainjection_patch.yaml - -configurations: - - kustomizeconfig.yaml -vars: - - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR - objref: - kind: Certificate - group: cert-manager.io - version: v1 - name: serving-cert # this name should match the one in certificate.yaml - fieldref: - fieldpath: metadata.namespace - - name: CERTIFICATE_NAME - objref: - kind: Certificate - group: cert-manager.io - version: v1 - name: serving-cert # this name should match the one in certificate.yaml - - name: SERVICE_NAMESPACE # namespace of the service - objref: - kind: Service - version: v1 - name: webhook-service - fieldref: - fieldpath: metadata.namespace - - name: SERVICE_NAME - objref: - kind: Service - version: v1 - name: webhook-service diff --git a/bootstrap/k3s/config/default/kustomizeconfig.yaml b/bootstrap/k3s/config/default/kustomizeconfig.yaml deleted file mode 100644 index eb191e64..00000000 --- a/bootstrap/k3s/config/default/kustomizeconfig.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# This configuration is for teaching kustomize how to update name ref and var substitution -varReference: -- kind: Deployment - path: spec/template/spec/volumes/secret/secretName diff --git a/bootstrap/k3s/config/default/manager_image_patch.yaml b/bootstrap/k3s/config/default/manager_image_patch.yaml deleted file mode 100644 index c17636d8..00000000 --- a/bootstrap/k3s/config/default/manager_image_patch.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - image: docker.io/kubespheredev/k3s-bootstrap-controller:main - name: manager diff --git a/bootstrap/k3s/config/default/manager_pull_policy.yaml b/bootstrap/k3s/config/default/manager_pull_policy.yaml deleted file mode 100644 index 74a0879c..00000000 --- a/bootstrap/k3s/config/default/manager_pull_policy.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - imagePullPolicy: Always diff --git a/bootstrap/k3s/config/default/manager_webhook_patch.yaml b/bootstrap/k3s/config/default/manager_webhook_patch.yaml deleted file mode 100644 index bccef6d7..00000000 --- a/bootstrap/k3s/config/default/manager_webhook_patch.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - ports: - - containerPort: 9443 - name: webhook-server - protocol: TCP - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - volumes: - - name: cert - secret: - secretName: $(SERVICE_NAME)-cert diff --git a/bootstrap/k3s/config/default/namespace.yaml b/bootstrap/k3s/config/default/namespace.yaml deleted file mode 100644 index 8b55c3cd..00000000 --- a/bootstrap/k3s/config/default/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: controller-manager - name: system diff --git a/bootstrap/k3s/config/default/webhookcainjection_patch.yaml b/bootstrap/k3s/config/default/webhookcainjection_patch.yaml deleted file mode 100644 index 04c08d02..00000000 --- a/bootstrap/k3s/config/default/webhookcainjection_patch.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: mutating-webhook-configuration - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validating-webhook-configuration - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/bootstrap/k3s/config/manager/kustomization.yaml b/bootstrap/k3s/config/manager/kustomization.yaml deleted file mode 100644 index 7394a6d0..00000000 --- a/bootstrap/k3s/config/manager/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: - - manager.yaml diff --git a/bootstrap/k3s/config/manager/manager.yaml b/bootstrap/k3s/config/manager/manager.yaml deleted file mode 100644 index 34ae98a9..00000000 --- a/bootstrap/k3s/config/manager/manager.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - control-plane: controller-manager -spec: - selector: - matchLabels: - control-plane: controller-manager - replicas: 1 - template: - metadata: - labels: - control-plane: controller-manager - spec: - containers: - - command: - - /manager - args: - - "--leader-elect" - - "--metrics-bind-addr=localhost:8080" - image: controller:latest - name: manager - ports: - - containerPort: 9440 - name: healthz - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: healthz - livenessProbe: - httpGet: - path: /healthz - port: healthz - terminationGracePeriodSeconds: 10 - serviceAccountName: manager - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane diff --git a/bootstrap/k3s/config/rbac/kustomization.yaml b/bootstrap/k3s/config/rbac/kustomization.yaml deleted file mode 100644 index 92215861..00000000 --- a/bootstrap/k3s/config/rbac/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -resources: -- service_account.yaml -- role.yaml -- role_binding.yaml -- leader_election_role.yaml -- leader_election_role_binding.yaml diff --git a/bootstrap/k3s/config/rbac/leader_election_role.yaml b/bootstrap/k3s/config/rbac/leader_election_role.yaml deleted file mode 100644 index 4190ec80..00000000 --- a/bootstrap/k3s/config/rbac/leader_election_role.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# permissions to do leader election. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: leader-election-role -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch diff --git a/bootstrap/k3s/config/rbac/leader_election_role_binding.yaml b/bootstrap/k3s/config/rbac/leader_election_role_binding.yaml deleted file mode 100644 index a73dfa95..00000000 --- a/bootstrap/k3s/config/rbac/leader_election_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: leader-election-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: leader-election-role -subjects: - - kind: ServiceAccount - name: manager - namespace: system diff --git a/bootstrap/k3s/config/rbac/role.yaml b/bootstrap/k3s/config/rbac/role.yaml deleted file mode 100644 index 9a5f9554..00000000 --- a/bootstrap/k3s/config/rbac/role.yaml +++ /dev/null @@ -1,49 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: manager-role -rules: -- apiGroups: - - "" - resources: - - configmaps - - events - - secrets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - bootstrap.cluster.x-k8s.io - resources: - - k3sconfigs - - k3sconfigs/finalizers - - k3sconfigs/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - cluster.x-k8s.io - resources: - - clusters - - clusters/status - - machinepools - - machinepools/status - - machines - - machines/status - - machinesets - verbs: - - get - - list - - watch diff --git a/bootstrap/k3s/config/rbac/role_binding.yaml b/bootstrap/k3s/config/rbac/role_binding.yaml deleted file mode 100644 index 3ffc9c2e..00000000 --- a/bootstrap/k3s/config/rbac/role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: manager-role -subjects: - - kind: ServiceAccount - name: manager - namespace: system diff --git a/bootstrap/k3s/config/rbac/service_account.yaml b/bootstrap/k3s/config/rbac/service_account.yaml deleted file mode 100644 index 77f747b5..00000000 --- a/bootstrap/k3s/config/rbac/service_account.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: manager - namespace: system diff --git a/bootstrap/k3s/config/samples/bootstrap_v1beta1_k3sconfig.yaml b/bootstrap/k3s/config/samples/bootstrap_v1beta1_k3sconfig.yaml deleted file mode 100644 index f2b353bc..00000000 --- a/bootstrap/k3s/config/samples/bootstrap_v1beta1_k3sconfig.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: K3sConfig -metadata: - name: k3sconfig-sample -spec: - # TODO(user): Add fields here diff --git a/bootstrap/k3s/config/samples/bootstrap_v1beta1_k3sconfigtemplate.yaml b/bootstrap/k3s/config/samples/bootstrap_v1beta1_k3sconfigtemplate.yaml deleted file mode 100644 index c6d495d4..00000000 --- a/bootstrap/k3s/config/samples/bootstrap_v1beta1_k3sconfigtemplate.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: K3sConfigTemplate -metadata: - name: k3sconfigtemplate-sample -spec: - # TODO(user): Add fields here diff --git a/bootstrap/k3s/config/samples/kustomization.yaml b/bootstrap/k3s/config/samples/kustomization.yaml deleted file mode 100644 index c60238ea..00000000 --- a/bootstrap/k3s/config/samples/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -## Append samples you want in your CSV to this file as resources ## -resources: -- bootstrap_v1beta1_k3sconfig.yaml -- bootstrap_v1beta1_k3sconfigtemplate.yaml -#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/bootstrap/k3s/config/scorecard/bases/config.yaml b/bootstrap/k3s/config/scorecard/bases/config.yaml deleted file mode 100644 index c7704784..00000000 --- a/bootstrap/k3s/config/scorecard/bases/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: scorecard.operatorframework.io/v1alpha3 -kind: Configuration -metadata: - name: config -stages: -- parallel: true - tests: [] diff --git a/bootstrap/k3s/config/scorecard/kustomization.yaml b/bootstrap/k3s/config/scorecard/kustomization.yaml deleted file mode 100644 index 50cd2d08..00000000 --- a/bootstrap/k3s/config/scorecard/kustomization.yaml +++ /dev/null @@ -1,16 +0,0 @@ -resources: -- bases/config.yaml -patchesJson6902: -- path: patches/basic.config.yaml - target: - group: scorecard.operatorframework.io - version: v1alpha3 - kind: Configuration - name: config -- path: patches/olm.config.yaml - target: - group: scorecard.operatorframework.io - version: v1alpha3 - kind: Configuration - name: config -#+kubebuilder:scaffold:patchesJson6902 diff --git a/bootstrap/k3s/config/scorecard/patches/basic.config.yaml b/bootstrap/k3s/config/scorecard/patches/basic.config.yaml deleted file mode 100644 index 4a6c8167..00000000 --- a/bootstrap/k3s/config/scorecard/patches/basic.config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - basic-check-spec - image: quay.io/operator-framework/scorecard-test:v1.22.2 - labels: - suite: basic - test: basic-check-spec-test diff --git a/bootstrap/k3s/config/scorecard/patches/olm.config.yaml b/bootstrap/k3s/config/scorecard/patches/olm.config.yaml deleted file mode 100644 index c342410a..00000000 --- a/bootstrap/k3s/config/scorecard/patches/olm.config.yaml +++ /dev/null @@ -1,50 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-bundle-validation - image: quay.io/operator-framework/scorecard-test:v1.22.2 - labels: - suite: olm - test: olm-bundle-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-validation - image: quay.io/operator-framework/scorecard-test:v1.22.2 - labels: - suite: olm - test: olm-crds-have-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-resources - image: quay.io/operator-framework/scorecard-test:v1.22.2 - labels: - suite: olm - test: olm-crds-have-resources-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-spec-descriptors - image: quay.io/operator-framework/scorecard-test:v1.22.2 - labels: - suite: olm - test: olm-spec-descriptors-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-status-descriptors - image: quay.io/operator-framework/scorecard-test:v1.22.2 - labels: - suite: olm - test: olm-status-descriptors-test diff --git a/bootstrap/k3s/config/webhook/kustomization.yaml b/bootstrap/k3s/config/webhook/kustomization.yaml deleted file mode 100644 index 9cf26134..00000000 --- a/bootstrap/k3s/config/webhook/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -resources: -- manifests.yaml -- service.yaml - -configurations: -- kustomizeconfig.yaml diff --git a/bootstrap/k3s/config/webhook/kustomizeconfig.yaml b/bootstrap/k3s/config/webhook/kustomizeconfig.yaml deleted file mode 100644 index 25e21e3c..00000000 --- a/bootstrap/k3s/config/webhook/kustomizeconfig.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# the following config is for teaching kustomize where to look at when substituting vars. -# It requires kustomize v2.1.0 or newer to work properly. -nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: MutatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/name - - kind: ValidatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/name - -namespace: -- kind: MutatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/namespace - create: true -- kind: ValidatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/namespace - create: true - -varReference: -- path: metadata/annotations diff --git a/bootstrap/k3s/config/webhook/manifests.yaml b/bootstrap/k3s/config/webhook/manifests.yaml deleted file mode 100644 index 8caf8749..00000000 --- a/bootstrap/k3s/config/webhook/manifests.yaml +++ /dev/null @@ -1,100 +0,0 @@ ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - creationTimestamp: null - name: mutating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-bootstrap-cluster-x-k8s-io-v1beta1-k3sconfig - failurePolicy: Fail - name: default.k3sconfig.bootstrap.cluster.x-k8s.io - rules: - - apiGroups: - - bootstrap.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - k3sconfigs - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-bootstrap-cluster-x-k8s-io-v1beta1-k3sconfigtemplate - failurePolicy: Fail - name: default.k3sconfigtemplate.bootstrap.cluster.x-k8s.io - rules: - - apiGroups: - - bootstrap.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - k3sconfigtemplates - sideEffects: None ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - creationTimestamp: null - name: validating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /validate-bootstrap-cluster-x-k8s-io-v1beta1-k3sconfig - failurePolicy: Fail - matchPolicy: Equivalent - name: validation.k3sconfig.bootstrap.cluster.x-k8s.io - rules: - - apiGroups: - - bootstrap.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - k3sconfigs - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /validate-bootstrap-cluster-x-k8s-io-v1beta1-k3sconfigtemplate - failurePolicy: Fail - matchPolicy: Equivalent - name: validation.k3sconfigtemplate.bootstrap.cluster.x-k8s.io - rules: - - apiGroups: - - bootstrap.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - k3sconfigtemplates - sideEffects: None diff --git a/bootstrap/k3s/config/webhook/service.yaml b/bootstrap/k3s/config/webhook/service.yaml deleted file mode 100644 index 3f638bd9..00000000 --- a/bootstrap/k3s/config/webhook/service.yaml +++ /dev/null @@ -1,13 +0,0 @@ - -apiVersion: v1 -kind: Service -metadata: - name: webhook-service - namespace: system -spec: - ports: - - port: 443 - protocol: TCP - targetPort: 9443 - selector: - control-plane: controller-manager diff --git a/bootstrap/k3s/controllers/doc.go b/bootstrap/k3s/controllers/doc.go deleted file mode 100644 index 98b7479a..00000000 --- a/bootstrap/k3s/controllers/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package controllers contains k3s config controllers. -package controllers diff --git a/bootstrap/k3s/controllers/k3sconfig_controller.go b/bootstrap/k3s/controllers/k3sconfig_controller.go deleted file mode 100644 index 5c953ca9..00000000 --- a/bootstrap/k3s/controllers/k3sconfig_controller.go +++ /dev/null @@ -1,794 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "context" - "fmt" - "time" - - "github.com/go-logr/logr" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - kerrors "k8s.io/apimachinery/pkg/util/errors" - bootstraputil "k8s.io/cluster-bootstrap/token/util" - "k8s.io/klog/v2" - "k8s.io/utils/pointer" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" - bsutil "sigs.k8s.io/cluster-api/bootstrap/util" - expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" - "sigs.k8s.io/cluster-api/feature" - "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/cluster-api/util/annotations" - "sigs.k8s.io/cluster-api/util/conditions" - "sigs.k8s.io/cluster-api/util/patch" - "sigs.k8s.io/cluster-api/util/predicates" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/source" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" - "github.com/kubesphere/kubekey/v3/bootstrap/k3s/pkg/cloudinit" - "github.com/kubesphere/kubekey/v3/bootstrap/k3s/pkg/locking" - k3stypes "github.com/kubesphere/kubekey/v3/bootstrap/k3s/pkg/types" - kklog "github.com/kubesphere/kubekey/v3/util/log" - "github.com/kubesphere/kubekey/v3/util/secret" -) - -// InitLocker is a lock that is used around kubeadm init. -type InitLocker interface { - Lock(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine) bool - Unlock(ctx context.Context, cluster *clusterv1.Cluster) bool -} - -// +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=k3sconfigs;k3sconfigs/status;k3sconfigs/finalizers,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status;machinesets;machines;machines/status;machinepools;machinepools/status,verbs=get;list;watch -// +kubebuilder:rbac:groups="",resources=secrets;events;configmaps,verbs=get;list;watch;create;update;patch;delete - -// K3sConfigReconciler reconciles a K3sConfig object -type K3sConfigReconciler struct { - client.Client - K3sInitLock InitLocker - - // WatchFilterValue is the label value used to filter events prior to reconciliation. - WatchFilterValue string -} - -// Scope is a scoped struct used during reconciliation. -type Scope struct { - logr.Logger - Config *infrabootstrapv1.K3sConfig - ConfigOwner *bsutil.ConfigOwner - Cluster *clusterv1.Cluster -} - -// SetupWithManager sets up the controller with the Manager. -func (r *K3sConfigReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { - if r.K3sInitLock == nil { - r.K3sInitLock = locking.NewControlPlaneInitMutex(mgr.GetClient()) - } - - b := ctrl.NewControllerManagedBy(mgr). - For(&infrabootstrapv1.K3sConfig{}). - WithOptions(options). - Watches( - &source.Kind{Type: &clusterv1.Machine{}}, - handler.EnqueueRequestsFromMapFunc(r.MachineToBootstrapMapFunc), - ).WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue)) - - if feature.Gates.Enabled(feature.MachinePool) { - b = b.Watches( - &source.Kind{Type: &expv1.MachinePool{}}, - handler.EnqueueRequestsFromMapFunc(r.MachinePoolToBootstrapMapFunc), - ).WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue)) - } - - c, err := b.Build(r) - if err != nil { - return errors.Wrap(err, "failed setting up with a controller manager") - } - - err = c.Watch( - &source.Kind{Type: &clusterv1.Cluster{}}, - handler.EnqueueRequestsFromMapFunc(r.ClusterToK3sConfigs), - predicates.All(ctrl.LoggerFrom(ctx), - predicates.ClusterUnpausedAndInfrastructureReady(ctrl.LoggerFrom(ctx)), - predicates.ResourceHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue), - ), - ) - if err != nil { - return errors.Wrap(err, "failed adding Watch for Clusters to controller manager") - } - - return nil -} - -// Reconcile handles K3sConfig events. -func (r *K3sConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, retErr error) { - log := ctrl.LoggerFrom(ctx) - - // Lookup the kubeadm config - config := &infrabootstrapv1.K3sConfig{} - if err := r.Client.Get(ctx, req.NamespacedName, config); err != nil { - if apierrors.IsNotFound(err) { - return ctrl.Result{}, nil - } - log.Error(err, "Failed to get config") - return ctrl.Result{}, err - } - - // AddOwners adds the owners of K3sConfig as k/v pairs to the logger. - // Specifically, it will add K3sControlPlane, MachineSet and MachineDeployment. - ctx, log, err := kklog.AddOwners(ctx, r.Client, config) - if err != nil { - return ctrl.Result{}, err - } - - // Look up the owner of this k3s config if there is one - configOwner, err := bsutil.GetConfigOwner(ctx, r.Client, config) - if apierrors.IsNotFound(err) { - // Could not find the owner yet, this is not an error and will rereconcile when the owner gets set. - return ctrl.Result{}, nil - } - if err != nil { - log.Error(err, "Failed to get owner") - return ctrl.Result{}, err - } - if configOwner == nil { - return ctrl.Result{}, nil - } - log = log.WithValues(configOwner.GetKind(), klog.KRef(configOwner.GetNamespace(), configOwner.GetName()), "resourceVersion", configOwner.GetResourceVersion()) - - log = log.WithValues("Cluster", klog.KRef(configOwner.GetNamespace(), configOwner.ClusterName())) - ctx = ctrl.LoggerInto(ctx, log) - - // Lookup the cluster the config owner is associated with - cluster, err := util.GetClusterByName(ctx, r.Client, configOwner.GetNamespace(), configOwner.ClusterName()) - if err != nil { - if errors.Cause(err) == util.ErrNoCluster { - log.Info(fmt.Sprintf("%s does not belong to a cluster yet, waiting until it's part of a cluster", configOwner.GetKind())) - return ctrl.Result{}, nil - } - - if apierrors.IsNotFound(err) { - log.Info("Cluster does not exist yet, waiting until it is created") - return ctrl.Result{}, nil - } - log.Error(err, "Could not get cluster with metadata") - return ctrl.Result{}, err - } - - if annotations.IsPaused(cluster, config) { - log.Info("Reconciliation is paused for this object") - return ctrl.Result{}, nil - } - - scope := &Scope{ - Logger: log, - Config: config, - ConfigOwner: configOwner, - Cluster: cluster, - } - - // Initialize the patch helper. - patchHelper, err := patch.NewHelper(config, r.Client) - if err != nil { - return ctrl.Result{}, err - } - - // Attempt to Patch the K3sConfig object and status after each reconciliation if no error occurs. - defer func() { - // always update the readyCondition; the summary is represented using the "1 of x completed" notation. - conditions.SetSummary(config, - conditions.WithConditions( - bootstrapv1.DataSecretAvailableCondition, - bootstrapv1.CertificatesAvailableCondition, - ), - ) - // Patch ObservedGeneration only if the reconciliation completed successfully - var patchOpts []patch.Option - if retErr == nil { - patchOpts = append(patchOpts, patch.WithStatusObservedGeneration{}) - } - if err := patchHelper.Patch(ctx, config, patchOpts...); err != nil { - log.Error(retErr, "Failed to patch config") - if retErr == nil { - retErr = err - } - } - }() - - switch { - // Wait for the infrastructure to be ready. - case !cluster.Status.InfrastructureReady: - log.Info("Cluster infrastructure is not ready, waiting") - conditions.MarkFalse(config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.WaitingForClusterInfrastructureReason, clusterv1.ConditionSeverityInfo, "") - return ctrl.Result{}, nil - // Reconcile status for machines that already have a secret reference, but our status isn't up-to-date. - // This case solves the pivoting scenario (or a backup restore) which doesn't preserve the status subresource on objects. - case configOwner.DataSecretName() != nil && (!config.Status.Ready || config.Status.DataSecretName == nil): - config.Status.Ready = true - config.Status.DataSecretName = configOwner.DataSecretName() - conditions.MarkTrue(config, bootstrapv1.DataSecretAvailableCondition) - return ctrl.Result{}, nil - // Status is ready means a config has been generated. - case config.Status.Ready: - return ctrl.Result{}, nil - } - - // Note: can't use IsFalse here because we need to handle the absence of the condition as well as false. - if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) { - return r.handleClusterNotInitialized(ctx, scope) - } - - // Every other case it's a join scenario - // Nb. in this case ClusterConfiguration and InitConfiguration should not be defined by users, but in case of misconfigurations, CABPK3s simply ignore them - - // Unlock any locks that might have been set during init process - r.K3sInitLock.Unlock(ctx, cluster) - - // if the .spec.cluster is missing, create a default one - if config.Spec.Cluster == nil { - log.Info("Creating default .spec.cluster") - config.Spec.Cluster = &infrabootstrapv1.Cluster{} - } - - // it's a control plane join - if configOwner.IsControlPlaneMachine() { - return r.joinControlplane(ctx, scope) - } - - // It's a worker join - return r.joinWorker(ctx, scope) -} - -func (r *K3sConfigReconciler) handleClusterNotInitialized(ctx context.Context, scope *Scope) (_ ctrl.Result, retErr error) { - // initialize the DataSecretAvailableCondition if missing. - // this is required in order to avoid the condition's LastTransitionTime to flicker in case of errors surfacing - // using the DataSecretGeneratedFailedReason - if conditions.GetReason(scope.Config, bootstrapv1.DataSecretAvailableCondition) != bootstrapv1.DataSecretGenerationFailedReason { - conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, clusterv1.WaitingForControlPlaneAvailableReason, clusterv1.ConditionSeverityInfo, "") - } - - // if it's NOT a control plane machine, requeue - if !scope.ConfigOwner.IsControlPlaneMachine() { - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil - } - - // if the machine has not ClusterConfiguration and InitConfiguration, requeue - if scope.Config.Spec.ServerConfiguration == nil && scope.Config.Spec.Cluster == nil { - scope.Info("Control plane is not ready, requeing joining control planes until ready.") - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil - } - - machine := &clusterv1.Machine{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(scope.ConfigOwner.Object, machine); err != nil { - return ctrl.Result{}, errors.Wrapf(err, "cannot convert %s to Machine", scope.ConfigOwner.GetKind()) - } - - // acquire the init lock so that only the first machine configured - // as control plane get processed here - // if not the first, requeue - if !r.K3sInitLock.Lock(ctx, scope.Cluster, machine) { - scope.Info("A control plane is already being initialized, requeing until control plane is ready") - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil - } - - defer func() { - if retErr != nil { - if !r.K3sInitLock.Unlock(ctx, scope.Cluster) { - retErr = kerrors.NewAggregate([]error{retErr, errors.New("failed to unlock the kubeadm init lock")}) - } - } - }() - - scope.Info("Creating BootstrapData for the first control plane") - - if scope.Config.Spec.ServerConfiguration == nil { - scope.Config.Spec.ServerConfiguration = &infrabootstrapv1.ServerConfiguration{} - } - - // injects into config.ClusterConfiguration values from top level object - r.reconcileTopLevelObjectSettings(ctx, scope.Cluster, machine, scope.Config) - - certificates := secret.NewCertificatesForInitialControlPlane() - err := certificates.LookupOrGenerate( - ctx, - r.Client, - util.ObjectKey(scope.Cluster), - *metav1.NewControllerRef(scope.Config, infrabootstrapv1.GroupVersion.WithKind("K3sConfig")), - ) - if err != nil { - conditions.MarkFalse(scope.Config, bootstrapv1.CertificatesAvailableCondition, bootstrapv1.CertificatesGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) - return ctrl.Result{}, err - } - - conditions.MarkTrue(scope.Config, bootstrapv1.CertificatesAvailableCondition) - - t, err := r.generateAndStoreToken(ctx, scope) - if err != nil { - return ctrl.Result{}, err - } - - initData, err := k3stypes.MarshalInitServerConfiguration(&scope.Config.Spec, t) - if err != nil { - scope.Error(err, "Failed to marshal server configuration") - return ctrl.Result{}, err - } - - files, err := r.resolveFiles(ctx, scope.Config) - if err != nil { - conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) - return ctrl.Result{}, err - } - - initConfigFile := bootstrapv1.File{ - Path: k3stypes.DefaultK3sConfigLocation, - Content: initData, - Owner: "root:root", - Permissions: "0640", - } - - controlPlaneInput := &cloudinit.ControlPlaneInput{ - BaseUserData: cloudinit.BaseUserData{ - AdditionalFiles: files, - PreK3sCommands: scope.Config.Spec.PreK3sCommands, - PostK3sCommands: scope.Config.Spec.PostK3sCommands, - ConfigFile: initConfigFile, - }, - Certificates: certificates, - } - - bootstrapInitData, err := cloudinit.NewInitControlPlane(controlPlaneInput) - if err != nil { - scope.Error(err, "Failed to generate user data for bootstrap control plane") - return ctrl.Result{}, err - } - - if err := r.storeBootstrapData(ctx, scope, bootstrapInitData); err != nil { - scope.Error(err, "Failed to store bootstrap data") - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil -} - -func (r *K3sConfigReconciler) joinWorker(ctx context.Context, scope *Scope) (ctrl.Result, error) { - scope.Info("Creating BootstrapData for the worker node") - - machine := &clusterv1.Machine{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(scope.ConfigOwner.Object, machine); err != nil { - return ctrl.Result{}, errors.Wrapf(err, "cannot convert %s to Machine", scope.ConfigOwner.GetKind()) - } - - // injects into config.Spec values from top level object - r.reconcileWorkerTopLevelObjectSettings(ctx, scope.Cluster, machine, scope.Config) - - // Ensure that agentConfiguration is properly set for joining node on the current cluster. - if res, err := r.reconcileDiscovery(ctx, scope.Cluster, scope.Config); err != nil { - return ctrl.Result{}, err - } else if !res.IsZero() { - return res, nil - } - - if scope.Config.Spec.AgentConfiguration == nil { - scope.Config.Spec.AgentConfiguration = &infrabootstrapv1.AgentConfiguration{} - } - - joinWorkerData, err := k3stypes.MarshalJoinAgentConfiguration(&scope.Config.Spec) - if err != nil { - scope.Error(err, "Failed to marshal join configuration") - return ctrl.Result{}, err - } - - files, err := r.resolveFiles(ctx, scope.Config) - if err != nil { - conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) - return ctrl.Result{}, err - } - - joinConfigFile := bootstrapv1.File{ - Path: k3stypes.DefaultK3sConfigLocation, - Content: joinWorkerData, - Owner: "root:root", - Permissions: "0640", - } - - workerJoinInput := &cloudinit.NodeInput{ - BaseUserData: cloudinit.BaseUserData{ - AdditionalFiles: files, - PreK3sCommands: scope.Config.Spec.PreK3sCommands, - PostK3sCommands: scope.Config.Spec.PostK3sCommands, - ConfigFile: joinConfigFile, - }, - } - - cloudInitData, err := cloudinit.NewNode(workerJoinInput) - if err != nil { - scope.Error(err, "Failed to generate user data for bootstrap control plane") - return ctrl.Result{}, err - } - - if err := r.storeBootstrapData(ctx, scope, cloudInitData); err != nil { - scope.Error(err, "Failed to store bootstrap data") - return ctrl.Result{}, err - } - return ctrl.Result{}, nil -} - -func (r *K3sConfigReconciler) joinControlplane(ctx context.Context, scope *Scope) (ctrl.Result, error) { - scope.Info("Creating BootstrapData for the joining control plane") - - if !scope.ConfigOwner.IsControlPlaneMachine() { - return ctrl.Result{}, fmt.Errorf("%s is not a valid control plane kind, only Machine is supported", scope.ConfigOwner.GetKind()) - } - - if scope.Config.Spec.ServerConfiguration == nil { - scope.Config.Spec.ServerConfiguration = &infrabootstrapv1.ServerConfiguration{} - } - - machine := &clusterv1.Machine{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(scope.ConfigOwner.Object, machine); err != nil { - return ctrl.Result{}, errors.Wrapf(err, "cannot convert %s to Machine", scope.ConfigOwner.GetKind()) - } - - // injects into config.ClusterConfiguration values from top level object - r.reconcileTopLevelObjectSettings(ctx, scope.Cluster, machine, scope.Config) - - // Ensure that joinConfiguration.Discovery is properly set for joining node on the current cluster. - if res, err := r.reconcileDiscovery(ctx, scope.Cluster, scope.Config); err != nil { - return ctrl.Result{}, err - } else if !res.IsZero() { - return res, nil - } - - joinData, err := k3stypes.MarshalJoinServerConfiguration(&scope.Config.Spec) - if err != nil { - scope.Error(err, "Failed to marshal join configuration") - return ctrl.Result{}, err - } - - files, err := r.resolveFiles(ctx, scope.Config) - if err != nil { - conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) - return ctrl.Result{}, err - } - - joinConfigFile := bootstrapv1.File{ - Path: k3stypes.DefaultK3sConfigLocation, - Content: joinData, - Owner: "root:root", - Permissions: "0640", - } - - controlPlaneJoinInput := &cloudinit.ControlPlaneInput{ - BaseUserData: cloudinit.BaseUserData{ - AdditionalFiles: files, - PreK3sCommands: scope.Config.Spec.PreK3sCommands, - PostK3sCommands: scope.Config.Spec.PostK3sCommands, - ConfigFile: joinConfigFile, - }, - } - - cloudInitData, err := cloudinit.NewJoinControlPlane(controlPlaneJoinInput) - if err != nil { - scope.Error(err, "Failed to generate user data for bootstrap control plane") - return ctrl.Result{}, err - } - - if err := r.storeBootstrapData(ctx, scope, cloudInitData); err != nil { - scope.Error(err, "Failed to store bootstrap data") - return ctrl.Result{}, err - } - return ctrl.Result{}, nil -} - -func (r *K3sConfigReconciler) generateAndStoreToken(ctx context.Context, scope *Scope) (string, error) { - t, err := bootstraputil.GenerateBootstrapToken() - if err != nil { - return "", errors.Wrap(err, "unable to generate bootstrap token") - } - - s := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-token", scope.Cluster.Name), - Namespace: scope.Config.Namespace, - Labels: map[string]string{ - clusterv1.ClusterLabelName: scope.Cluster.Name, - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: infrabootstrapv1.GroupVersion.String(), - Kind: "K3sConfig", - Name: scope.Config.Name, - UID: scope.Config.UID, - Controller: pointer.Bool(true), - }, - }, - }, - Data: map[string][]byte{ - "value": []byte(t), - }, - Type: clusterv1.ClusterSecretType, - } - - // as secret creation and scope.Config status patch are not atomic operations - // it is possible that secret creation happens but the config.Status patches are not applied - if err := r.Client.Create(ctx, s); err != nil { - if !apierrors.IsAlreadyExists(err) { - return "", errors.Wrapf(err, "failed to create token for K3sConfig %s/%s", scope.Config.Namespace, scope.Config.Name) - } - if err := r.Client.Update(ctx, s); err != nil { - return "", errors.Wrapf(err, "failed to update bootstrap token secret for K3sConfig %s/%s", scope.Config.Namespace, scope.Config.Name) - } - } - - return t, nil -} - -// resolveFiles maps .Spec.Files into cloudinit.Files, resolving any object references -// along the way. -func (r *K3sConfigReconciler) resolveFiles(ctx context.Context, cfg *infrabootstrapv1.K3sConfig) ([]bootstrapv1.File, error) { - collected := make([]bootstrapv1.File, 0, len(cfg.Spec.Files)) - - for i := range cfg.Spec.Files { - in := cfg.Spec.Files[i] - if in.ContentFrom != nil { - data, err := r.resolveSecretFileContent(ctx, cfg.Namespace, in) - if err != nil { - return nil, errors.Wrapf(err, "failed to resolve file source") - } - in.ContentFrom = nil - in.Content = string(data) - } - collected = append(collected, in) - } - - return collected, nil -} - -// resolveSecretFileContent returns file content fetched from a referenced secret object. -func (r *K3sConfigReconciler) resolveSecretFileContent(ctx context.Context, ns string, source bootstrapv1.File) ([]byte, error) { - s := &corev1.Secret{} - key := types.NamespacedName{Namespace: ns, Name: source.ContentFrom.Secret.Name} - if err := r.Client.Get(ctx, key, s); err != nil { - if apierrors.IsNotFound(err) { - return nil, errors.Wrapf(err, "secret not found: %s", key) - } - return nil, errors.Wrapf(err, "failed to retrieve Secret %q", key) - } - data, ok := s.Data[source.ContentFrom.Secret.Key] - if !ok { - return nil, errors.Errorf("secret references non-existent secret key: %q", source.ContentFrom.Secret.Key) - } - return data, nil -} - -// storeBootstrapData creates a new secret with the data passed in as input, -// sets the reference in the configuration status and ready to true. -func (r *K3sConfigReconciler) storeBootstrapData(ctx context.Context, scope *Scope, data []byte) error { - log := ctrl.LoggerFrom(ctx) - - s := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: scope.Config.Name, - Namespace: scope.Config.Namespace, - Labels: map[string]string{ - clusterv1.ClusterLabelName: scope.Cluster.Name, - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: infrabootstrapv1.GroupVersion.String(), - Kind: "K3sConfig", - Name: scope.Config.Name, - UID: scope.Config.UID, - Controller: pointer.Bool(true), - }, - }, - }, - Data: map[string][]byte{ - "value": data, - }, - Type: clusterv1.ClusterSecretType, - } - - // as secret creation and scope.Config status patch are not atomic operations - // it is possible that secret creation happens but the config.Status patches are not applied - if err := r.Client.Create(ctx, s); err != nil { - if !apierrors.IsAlreadyExists(err) { - return errors.Wrapf(err, "failed to create bootstrap data secret for K3sConfig %s/%s", scope.Config.Namespace, scope.Config.Name) - } - log.Info("bootstrap data secret for K3sConfig already exists, updating", "Secret", klog.KObj(s)) - if err := r.Client.Update(ctx, s); err != nil { - return errors.Wrapf(err, "failed to update bootstrap data secret for K3sConfig %s/%s", scope.Config.Namespace, scope.Config.Name) - } - } - scope.Config.Status.DataSecretName = pointer.String(s.Name) - scope.Config.Status.Ready = true - conditions.MarkTrue(scope.Config, bootstrapv1.DataSecretAvailableCondition) - return nil -} - -func (r *K3sConfigReconciler) reconcileDiscovery(ctx context.Context, cluster *clusterv1.Cluster, config *infrabootstrapv1.K3sConfig) (ctrl.Result, error) { - log := ctrl.LoggerFrom(ctx) - - // if config already contains a file discovery configuration, respect it without further validations - if config.Spec.Cluster.TokenFile != "" { - return ctrl.Result{}, nil - } - - // if BootstrapToken already contains an APIServerEndpoint, respect it; otherwise inject the APIServerEndpoint endpoint defined in cluster status - apiServerEndpoint := config.Spec.Cluster.Server - if apiServerEndpoint == "" { - if !cluster.Spec.ControlPlaneEndpoint.IsValid() { - log.V(1).Info("Waiting for Cluster Controller to set Cluster.Server") - return ctrl.Result{RequeueAfter: 10 * time.Second}, nil - } - - apiServerEndpoint = cluster.Spec.ControlPlaneEndpoint.String() - config.Spec.Cluster.Server = fmt.Sprintf("https://%s", apiServerEndpoint) - log.V(3).Info("Altering Cluster.Server", "Server", apiServerEndpoint) - } - - // if BootstrapToken already contains a token, respect it; otherwise create a new bootstrap token for the node to join - if config.Spec.Cluster.Token == "" { - s := &corev1.Secret{} - obj := client.ObjectKey{ - Namespace: config.Namespace, - Name: fmt.Sprintf("%s-token", cluster.Name), - } - - if err := r.Client.Get(ctx, obj, s); err != nil { - return ctrl.Result{}, errors.Wrapf(err, "failed to get token for K3sConfig %s/%s", config.Namespace, config.Name) - } - - config.Spec.Cluster.Token = string(s.Data["value"]) - log.V(3).Info("Altering Cluster.Token") - } - - return ctrl.Result{}, nil -} - -// MachineToBootstrapMapFunc is a handler.ToRequestsFunc to be used to enqueue -// request for reconciliation of K3sConfig. -func (r *K3sConfigReconciler) MachineToBootstrapMapFunc(o client.Object) []ctrl.Request { - m, ok := o.(*clusterv1.Machine) - if !ok { - panic(fmt.Sprintf("Expected a Machine but got a %T", o)) - } - - var result []ctrl.Request - if m.Spec.Bootstrap.ConfigRef != nil && m.Spec.Bootstrap.ConfigRef.GroupVersionKind() == infrabootstrapv1.GroupVersion.WithKind("K3sConfig") { - name := client.ObjectKey{Namespace: m.Namespace, Name: m.Spec.Bootstrap.ConfigRef.Name} - result = append(result, ctrl.Request{NamespacedName: name}) - } - return result -} - -// MachinePoolToBootstrapMapFunc is a handler.ToRequestsFunc to be used to enqueue -// request for reconciliation of K3sConfig. -func (r *K3sConfigReconciler) MachinePoolToBootstrapMapFunc(o client.Object) []ctrl.Request { - m, ok := o.(*expv1.MachinePool) - if !ok { - panic(fmt.Sprintf("Expected a MachinePool but got a %T", o)) - } - - var result []ctrl.Request - configRef := m.Spec.Template.Spec.Bootstrap.ConfigRef - if configRef != nil && configRef.GroupVersionKind().GroupKind() == infrabootstrapv1.GroupVersion.WithKind("K3sConfig").GroupKind() { - name := client.ObjectKey{Namespace: m.Namespace, Name: configRef.Name} - result = append(result, ctrl.Request{NamespacedName: name}) - } - return result -} - -// ClusterToK3sConfigs is a handler.ToRequestsFunc to be used to enqueue -// requests for reconciliation of K3sConfig. -func (r *K3sConfigReconciler) ClusterToK3sConfigs(o client.Object) []ctrl.Request { - var result []ctrl.Request - - c, ok := o.(*clusterv1.Cluster) - if !ok { - panic(fmt.Sprintf("Expected a Cluster but got a %T", o)) - } - - selectors := []client.ListOption{ - client.InNamespace(c.Namespace), - client.MatchingLabels{ - clusterv1.ClusterLabelName: c.Name, - }, - } - - machineList := &clusterv1.MachineList{} - if err := r.Client.List(context.TODO(), machineList, selectors...); err != nil { - return nil - } - - for _, m := range machineList.Items { - if m.Spec.Bootstrap.ConfigRef != nil && - m.Spec.Bootstrap.ConfigRef.GroupVersionKind().GroupKind() == infrabootstrapv1.GroupVersion.WithKind("K3sConfig").GroupKind() { - name := client.ObjectKey{Namespace: m.Namespace, Name: m.Spec.Bootstrap.ConfigRef.Name} - result = append(result, ctrl.Request{NamespacedName: name}) - } - } - - if feature.Gates.Enabled(feature.MachinePool) { - machinePoolList := &expv1.MachinePoolList{} - if err := r.Client.List(context.TODO(), machinePoolList, selectors...); err != nil { - return nil - } - - for _, mp := range machinePoolList.Items { - if mp.Spec.Template.Spec.Bootstrap.ConfigRef != nil && - mp.Spec.Template.Spec.Bootstrap.ConfigRef.GroupVersionKind().GroupKind() == infrabootstrapv1.GroupVersion.WithKind("K3sConfig").GroupKind() { - name := client.ObjectKey{Namespace: mp.Namespace, Name: mp.Spec.Template.Spec.Bootstrap.ConfigRef.Name} - result = append(result, ctrl.Request{NamespacedName: name}) - } - } - } - - return result -} - -// reconcileTopLevelObjectSettings injects into config.ClusterConfiguration values from top level objects like cluster and machine. -// The implementation func respect user provided config values, but in case some of them are missing, values from top level objects are used. -func (r *K3sConfigReconciler) reconcileTopLevelObjectSettings(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine, config *infrabootstrapv1.K3sConfig) { - log := ctrl.LoggerFrom(ctx) - - // If there are no Network settings defined in ClusterConfiguration, use ClusterNetwork settings, if defined - if cluster.Spec.ClusterNetwork != nil { - if config.Spec.ServerConfiguration.Networking.ClusterDomain == "" && cluster.Spec.ClusterNetwork.ServiceDomain != "" { - config.Spec.ServerConfiguration.Networking.ClusterDomain = cluster.Spec.ClusterNetwork.ServiceDomain - log.V(3).Info("Altering ServerConfiguration.Networking.ClusterDomain", "ClusterDomain", config.Spec.ServerConfiguration.Networking.ClusterDomain) - } - if config.Spec.ServerConfiguration.Networking.ServiceCIDR == "" && - cluster.Spec.ClusterNetwork.Services != nil && - len(cluster.Spec.ClusterNetwork.Services.CIDRBlocks) > 0 { - config.Spec.ServerConfiguration.Networking.ServiceCIDR = cluster.Spec.ClusterNetwork.Services.String() - log.V(3).Info("Altering ServerConfiguration.Networking.ServiceCIDR", "ServiceCIDR", config.Spec.ServerConfiguration.Networking.ServiceCIDR) - } - if config.Spec.ServerConfiguration.Networking.ClusterCIDR == "" && - cluster.Spec.ClusterNetwork.Pods != nil && - len(cluster.Spec.ClusterNetwork.Pods.CIDRBlocks) > 0 { - config.Spec.ServerConfiguration.Networking.ClusterCIDR = cluster.Spec.ClusterNetwork.Pods.String() - log.V(3).Info("Altering ServerConfiguration.Networking.ClusterCIDR", "ClusterCIDR", config.Spec.ServerConfiguration.Networking.ClusterCIDR) - } - } - - // If there are no Version settings defined, use Version from machine, if defined - if config.Spec.Version == "" && machine.Spec.Version != nil { - config.Spec.Version = *machine.Spec.Version - log.V(3).Info("Altering Spec.Version", "Version", config.Spec.Version) - } -} - -func (r *K3sConfigReconciler) reconcileWorkerTopLevelObjectSettings(ctx context.Context, _ *clusterv1.Cluster, machine *clusterv1.Machine, config *infrabootstrapv1.K3sConfig) { - log := ctrl.LoggerFrom(ctx) - - // If there are no Version settings defined, use Version from machine, if defined - if config.Spec.Version == "" && machine.Spec.Version != nil { - config.Spec.Version = *machine.Spec.Version - log.V(3).Info("Altering Spec.Version", "Version", config.Spec.Version) - } -} diff --git a/bootstrap/k3s/controllers/suite_test.go b/bootstrap/k3s/controllers/suite_test.go deleted file mode 100644 index 76505fda..00000000 --- a/bootstrap/k3s/controllers/suite_test.go +++ /dev/null @@ -1,29 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -//var ( -// env *envtest.Environment -// ctx = ctrl.SetupSignalHandler() -//) -// -//func TestMain(m *testing.M) { -// os.Exit(envtest.Run(ctx, envtest.RunInput{ -// M: m, -// SetupEnv: func(e *envtest.Environment) { env = e }, -// })) -//} diff --git a/bootstrap/k3s/hack/boilerplate.go.txt b/bootstrap/k3s/hack/boilerplate.go.txt deleted file mode 100644 index 62802d18..00000000 --- a/bootstrap/k3s/hack/boilerplate.go.txt +++ /dev/null @@ -1,15 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ \ No newline at end of file diff --git a/bootstrap/k3s/main.go b/bootstrap/k3s/main.go deleted file mode 100644 index a994a1fe..00000000 --- a/bootstrap/k3s/main.go +++ /dev/null @@ -1,193 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package main -package main - -import ( - "flag" - "fmt" - "math/rand" - "os" - "time" - - "github.com/spf13/pflag" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth" - "k8s.io/client-go/tools/leaderelection/resourcelock" - "k8s.io/klog/v2" - "k8s.io/klog/v2/klogr" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/remote" - expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" - "sigs.k8s.io/cluster-api/feature" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/healthz" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" - "github.com/kubesphere/kubekey/v3/bootstrap/k3s/controllers" - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" - //+kubebuilder:scaffold:imports -) - -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") -) - -func init() { - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(clusterv1.AddToScheme(scheme)) - utilruntime.Must(expv1.AddToScheme(scheme)) - utilruntime.Must(infrabootstrapv1.AddToScheme(scheme)) - utilruntime.Must(infracontrolplanev1.AddToScheme(scheme)) - //+kubebuilder:scaffold:scheme -} - -var ( - metricsAddr string - enableLeaderElection bool - leaderElectionLeaseDuration time.Duration - leaderElectionRenewDeadline time.Duration - leaderElectionRetryPeriod time.Duration - k3sConfigConcurrency int - healthAddr string - watchFilterValue string - watchNamespace string - syncPeriod time.Duration - webhookPort int - webhookCertDir string -) - -func main() { - klog.InitFlags(nil) - - rand.Seed(time.Now().UnixNano()) - initFlags(pflag.CommandLine) - pflag.CommandLine.AddGoFlagSet(flag.CommandLine) - pflag.Parse() - - ctrl.SetLogger(klogr.New()) - - ctx := ctrl.SetupSignalHandler() - - restConfig := ctrl.GetConfigOrDie() - restConfig.UserAgent = remote.DefaultClusterAPIUserAgent("cluster-api-k3s-bootstrap-manager") - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "k3s-bootstrap-manager-leader-election-capkk", - LeaderElectionResourceLock: resourcelock.LeasesResourceLock, - LeaseDuration: &leaderElectionLeaseDuration, - RenewDeadline: &leaderElectionRenewDeadline, - RetryPeriod: &leaderElectionRetryPeriod, - SyncPeriod: &syncPeriod, - ClientDisableCacheFor: []client.Object{ - &corev1.ConfigMap{}, - &corev1.Secret{}, - }, - Namespace: watchNamespace, - Port: webhookPort, - HealthProbeBindAddress: healthAddr, - CertDir: webhookCertDir, - }) - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) - } - - if err := (&controllers.K3sConfigReconciler{ - Client: mgr.GetClient(), - WatchFilterValue: watchFilterValue, - }).SetupWithManager(ctx, mgr, concurrency(k3sConfigConcurrency)); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "K3sConfig") - os.Exit(1) - } - - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") - os.Exit(1) - } - - if err = (&infrabootstrapv1.K3sConfig{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "K3sConfig") - os.Exit(1) - } - if err = (&infrabootstrapv1.K3sConfigTemplate{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "K3sConfigTemplate") - os.Exit(1) - } - // +kubebuilder:scaffold:builder - setupLog.Info("starting manager") - if err := mgr.Start(ctx); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) - } -} - -func initFlags(fs *pflag.FlagSet) { - fs.StringVar(&metricsAddr, "metrics-bind-addr", "localhost:8080", - "The address the metric endpoint binds to.") - - fs.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") - - fs.DurationVar(&leaderElectionLeaseDuration, "leader-elect-lease-duration", 15*time.Second, - "Interval at which non-leader candidates will wait to force acquire leadership (duration string)") - - fs.DurationVar(&leaderElectionRenewDeadline, "leader-elect-renew-deadline", 10*time.Second, - "Duration that the leading controller manager will retry refreshing leadership before giving up (duration string)") - - fs.DurationVar(&leaderElectionRetryPeriod, "leader-elect-retry-period", 2*time.Second, - "Duration the LeaderElector clients should wait between tries of actions (duration string)") - - fs.StringVar(&watchNamespace, "namespace", "", - "Namespace that the controller watches to reconcile cluster-api objects. If unspecified, the controller watches for cluster-api objects across all namespaces.") - - fs.StringVar(&healthAddr, "health-addr", ":9440", - "The address the health endpoint binds to.") - - fs.IntVar(&k3sConfigConcurrency, "k3sconfig-concurrency", 10, - "Number of kubeadm configs to process simultaneously") - - fs.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, - "The minimum interval at which watched resources are reconciled (e.g. 15m)") - - fs.StringVar(&watchFilterValue, "watch-filter", "", - fmt.Sprintf("Label value that the controller watches to reconcile cluster-api objects. Label key is always %s. If unspecified, the controller watches for all cluster-api objects.", clusterv1.WatchLabel)) - - fs.IntVar(&webhookPort, "webhook-port", 9443, - "Webhook Server port") - - fs.StringVar(&webhookCertDir, "webhook-cert-dir", "/tmp/k8s-webhook-server/serving-certs/", - "Webhook cert dir, only used when webhook-port is specified.") - - feature.MutableGates.AddFlag(fs) -} - -func concurrency(c int) controller.Options { - return controller.Options{MaxConcurrentReconciles: c} -} diff --git a/bootstrap/k3s/pkg/cloudinit/cloudinit.go b/bootstrap/k3s/pkg/cloudinit/cloudinit.go deleted file mode 100644 index 098d9492..00000000 --- a/bootstrap/k3s/pkg/cloudinit/cloudinit.go +++ /dev/null @@ -1,76 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "bytes" - _ "embed" - "text/template" - - "github.com/pkg/errors" - bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" -) - -const ( - // sentinelFileCommand writes a file to /run/cluster-api to signal successful Kubernetes bootstrapping in a way that - // works both for Linux and Windows OS. - sentinelFileCommand = "echo success > /run/cluster-api/bootstrap-success.complete" - cloudConfigHeader = `## template: jinja -#cloud-config -` -) - -// BaseUserData is shared across all the various types of files written to disk. -type BaseUserData struct { - Header string - PreK3sCommands []string - PostK3sCommands []string - AdditionalFiles []bootstrapv1.File - WriteFiles []bootstrapv1.File - ConfigFile bootstrapv1.File - SentinelFileCommand string -} - -func (input *BaseUserData) prepare() { - input.Header = cloudConfigHeader - input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...) - input.WriteFiles = append(input.WriteFiles, input.ConfigFile) - input.SentinelFileCommand = sentinelFileCommand -} - -func generate(kind string, tpl string, data interface{}) ([]byte, error) { - tm := template.New(kind).Funcs(defaultTemplateFuncMap) - if _, err := tm.Parse(filesTemplate); err != nil { - return nil, errors.Wrap(err, "failed to parse files template") - } - - if _, err := tm.Parse(commandsTemplate); err != nil { - return nil, errors.Wrap(err, "failed to parse commands template") - } - - t, err := tm.Parse(tpl) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse %s template", kind) - } - - var out bytes.Buffer - if err := t.Execute(&out, data); err != nil { - return nil, errors.Wrapf(err, "failed to generate %s template", kind) - } - - return out.Bytes(), nil -} diff --git a/bootstrap/k3s/pkg/cloudinit/commands.go b/bootstrap/k3s/pkg/cloudinit/commands.go deleted file mode 100644 index 9fefb6d9..00000000 --- a/bootstrap/k3s/pkg/cloudinit/commands.go +++ /dev/null @@ -1,26 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -const ( - commandsTemplate = `{{- define "commands" -}} -{{ range . }} - - {{printf "%q" .}} -{{- end -}} -{{- end -}} -` -) diff --git a/bootstrap/k3s/pkg/cloudinit/controlplane_init.go b/bootstrap/k3s/pkg/cloudinit/controlplane_init.go deleted file mode 100644 index dc4d3b6c..00000000 --- a/bootstrap/k3s/pkg/cloudinit/controlplane_init.go +++ /dev/null @@ -1,58 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "github.com/kubesphere/kubekey/v3/util/secret" -) - -const ( - controlPlaneCloudInit = `{{.Header}} -{{template "files" .WriteFiles}} -- path: /run/cluster-api/placeholder - owner: root:root - permissions: '0640' - content: "This placeholder file is used to create the /run/cluster-api sub directory in a way that is compatible with both Linux and Windows (mkdir -p /run/cluster-api does not work with Windows)" -runcmd: -{{- template "commands" .PreK3sCommands }} - - "INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_EXEC='server' /usr/local/bin/k3s-install.sh" -{{- template "commands" .PostK3sCommands }} -` -) - -// ControlPlaneInput defines the context to generate a controlplane instance user data. -type ControlPlaneInput struct { - BaseUserData - secret.Certificates - - ServerConfiguration string -} - -// NewInitControlPlane returns the clouding string to be used on initializing a controlplane instance. -func NewInitControlPlane(input *ControlPlaneInput) ([]byte, error) { - input.Header = cloudConfigHeader - input.WriteFiles = input.Certificates.AsFiles() - input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...) - input.WriteFiles = append(input.WriteFiles, input.ConfigFile) - input.SentinelFileCommand = sentinelFileCommand - userData, err := generate("InitControlplane", controlPlaneCloudInit, input) - if err != nil { - return nil, err - } - - return userData, nil -} diff --git a/bootstrap/k3s/pkg/cloudinit/controlplane_join.go b/bootstrap/k3s/pkg/cloudinit/controlplane_join.go deleted file mode 100644 index 88ea6146..00000000 --- a/bootstrap/k3s/pkg/cloudinit/controlplane_join.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "github.com/pkg/errors" -) - -const ( - controlPlaneJoinCloudInit = `{{.Header}} -{{template "files" .WriteFiles}} -- path: /run/cluster-api/placeholder - owner: root:root - permissions: '0640' - content: "This placeholder file is used to create the /run/cluster-api sub directory in a way that is compatible with both Linux and Windows (mkdir -p /run/cluster-api does not work with Windows)" -runcmd: -{{- template "commands" .PreK3sCommands }} - - "INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_EXEC='server' /usr/local/bin/k3s-install.sh" -{{- template "commands" .PostK3sCommands }} -` -) - -// NewJoinControlPlane returns the cloudinit string to be used on joining a control plane instance. -func NewJoinControlPlane(input *ControlPlaneInput) ([]byte, error) { - input.prepare() - userData, err := generate("JoinControlplane", controlPlaneJoinCloudInit, input) - if err != nil { - return nil, errors.Wrapf(err, "failed to generate user data for machine joining control plane") - } - - return userData, err -} diff --git a/bootstrap/k3s/pkg/cloudinit/doc.go b/bootstrap/k3s/pkg/cloudinit/doc.go deleted file mode 100644 index 1a473c7d..00000000 --- a/bootstrap/k3s/pkg/cloudinit/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package cloudinit implements kubeadm cloudinit functionality. -package cloudinit diff --git a/bootstrap/k3s/pkg/cloudinit/files.go b/bootstrap/k3s/pkg/cloudinit/files.go deleted file mode 100644 index fd0c3f89..00000000 --- a/bootstrap/k3s/pkg/cloudinit/files.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -const ( - filesTemplate = `{{ define "files" -}} -write_files:{{ range . }} -- path: {{.Path}} - {{ if ne .Encoding "" -}} - encoding: "{{.Encoding}}" - {{ end -}} - {{ if ne .Owner "" -}} - owner: {{.Owner}} - {{ end -}} - {{ if ne .Permissions "" -}} - permissions: '{{.Permissions}}' - {{ end -}} - {{ if .Append -}} - append: true - {{ end -}} - content: | -{{.Content | Indent 6}} -{{- end -}} -{{- end -}} -` -) diff --git a/bootstrap/k3s/pkg/cloudinit/node.go b/bootstrap/k3s/pkg/cloudinit/node.go deleted file mode 100644 index 0335c9b9..00000000 --- a/bootstrap/k3s/pkg/cloudinit/node.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "github.com/pkg/errors" -) - -const ( - workerCloudInit = `{{.Header}} -{{template "files" .WriteFiles}} -- path: /run/cluster-api/placeholder - owner: root:root - permissions: '0640' - content: "This placeholder file is used to create the /run/cluster-api sub directory in a way that is compatible with both Linux and Windows (mkdir -p /run/cluster-api does not work with Windows)" -runcmd: -{{- template "commands" .PreK3sCommands }} - - "INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_EXEC='agent' /usr/local/bin/k3s-install.sh" -{{- template "commands" .PostK3sCommands }} -` -) - -// NodeInput defines the context to generate an agent node cloud-init. -type NodeInput struct { - BaseUserData -} - -// NewNode returns the cloud-init for joining a node instance. -func NewNode(input *NodeInput) ([]byte, error) { - input.prepare() - userData, err := generate("JoinWorker", workerCloudInit, input) - if err != nil { - return nil, errors.Wrapf(err, "failed to generate user data for machine joining worker node") - } - - return userData, err -} diff --git a/bootstrap/k3s/pkg/cloudinit/utils.go b/bootstrap/k3s/pkg/cloudinit/utils.go deleted file mode 100644 index 1bfd0037..00000000 --- a/bootstrap/k3s/pkg/cloudinit/utils.go +++ /dev/null @@ -1,34 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "strings" - "text/template" -) - -var ( - defaultTemplateFuncMap = template.FuncMap{ - "Indent": templateYAMLIndent, - } -) - -func templateYAMLIndent(i int, input string) string { - split := strings.Split(input, "\n") - ident := "\n" + strings.Repeat(" ", i) - return strings.Repeat(" ", i) + strings.Join(split, ident) -} diff --git a/bootstrap/k3s/pkg/locking/control_plane_init_mutex.go b/bootstrap/k3s/pkg/locking/control_plane_init_mutex.go deleted file mode 100644 index fdc71822..00000000 --- a/bootstrap/k3s/pkg/locking/control_plane_init_mutex.go +++ /dev/null @@ -1,190 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package locking implements locking functionality. -package locking - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog/v2" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const semaphoreInformationKey = "lock-information" - -// ControlPlaneInitMutex uses a ConfigMap to synchronize cluster initialization. -type ControlPlaneInitMutex struct { - client client.Client -} - -// NewControlPlaneInitMutex returns a lock that can be held by a control plane node before init. -func NewControlPlaneInitMutex(client client.Client) *ControlPlaneInitMutex { - return &ControlPlaneInitMutex{ - client: client, - } -} - -// Lock allows a control plane node to be the first and only node to run kubeadm init. -func (c *ControlPlaneInitMutex) Lock(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine) bool { - sema := newSemaphore() - cmName := configMapName(cluster.Name) - log := ctrl.LoggerFrom(ctx, "ConfigMap", klog.KRef(cluster.Namespace, cmName)) - err := c.client.Get(ctx, client.ObjectKey{ - Namespace: cluster.Namespace, - Name: cmName, - }, sema.ConfigMap) - switch { - case apierrors.IsNotFound(err): - break - case err != nil: - log.Error(err, "Failed to acquire init lock") - return false - default: // Successfully found an existing config map. - info, err := sema.information() - if err != nil { - log.Error(err, "Failed to get information about the existing init lock") - return false - } - // The machine requesting the lock is the machine that created the lock, therefore the lock is acquired. - if info.MachineName == machine.Name { - return true - } - - // If the machine that created the lock can not be found unlock the mutex. - if err := c.client.Get(ctx, client.ObjectKey{ - Namespace: cluster.Namespace, - Name: info.MachineName, - }, &clusterv1.Machine{}); err != nil { - log.Error(err, "Failed to get machine holding init lock") - if apierrors.IsNotFound(err) { - c.Unlock(ctx, cluster) - } - } - log.Info(fmt.Sprintf("Waiting for Machine %s to initialize", info.MachineName)) - return false - } - - // Adds owner reference, namespace and name - sema.setMetadata(cluster) - // Adds the additional information - if err := sema.setInformation(&information{MachineName: machine.Name}); err != nil { - log.Error(err, "Failed to acquire init lock while setting semaphore information") - return false - } - - log.Info("Attempting to acquire the lock") - err = c.client.Create(ctx, sema.ConfigMap) - switch { - case apierrors.IsAlreadyExists(err): - log.Info("Cannot acquire the init lock. The init lock has been acquired by someone else") - return false - case err != nil: - log.Error(err, "Error acquiring the init lock") - return false - default: - return true - } -} - -// Unlock releases the lock. -func (c *ControlPlaneInitMutex) Unlock(ctx context.Context, cluster *clusterv1.Cluster) bool { - sema := newSemaphore() - cmName := configMapName(cluster.Name) - log := ctrl.LoggerFrom(ctx, "ConfigMap", klog.KRef(cluster.Namespace, cmName)) - err := c.client.Get(ctx, client.ObjectKey{ - Namespace: cluster.Namespace, - Name: cmName, - }, sema.ConfigMap) - switch { - case apierrors.IsNotFound(err): - log.Info("Control plane init lock not found, it may have been released already") - return true - case err != nil: - log.Error(err, "Error unlocking the control plane init lock") - return false - default: - // Delete the config map semaphore if there is no error fetching it - if err := c.client.Delete(ctx, sema.ConfigMap); err != nil { - if apierrors.IsNotFound(err) { - return true - } - log.Error(err, "Error deleting the config map underlying the control plane init lock") - return false - } - return true - } -} - -type information struct { - MachineName string `json:"machineName"` -} - -type semaphore struct { - *corev1.ConfigMap -} - -func newSemaphore() *semaphore { - return &semaphore{&corev1.ConfigMap{}} -} - -func configMapName(clusterName string) string { - return fmt.Sprintf("%s-lock", clusterName) -} - -func (s semaphore) information() (*information, error) { - li := &information{} - if err := json.Unmarshal([]byte(s.Data[semaphoreInformationKey]), li); err != nil { - return nil, errors.Wrap(err, "failed to unmarshal semaphore information") - } - return li, nil -} - -func (s semaphore) setInformation(information *information) error { - b, err := json.Marshal(information) - if err != nil { - return errors.Wrap(err, "failed to marshal semaphore information") - } - s.Data = map[string]string{} - s.Data[semaphoreInformationKey] = string(b) - return nil -} - -func (s *semaphore) setMetadata(cluster *clusterv1.Cluster) { - s.ObjectMeta = metav1.ObjectMeta{ - Namespace: cluster.Namespace, - Name: configMapName(cluster.Name), - Labels: map[string]string{ - clusterv1.ClusterLabelName: cluster.Name, - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: cluster.APIVersion, - Kind: cluster.Kind, - Name: cluster.Name, - UID: cluster.UID, - }, - }, - } -} diff --git a/bootstrap/k3s/pkg/locking/control_plane_init_mutex_test.go b/bootstrap/k3s/pkg/locking/control_plane_init_mutex_test.go deleted file mode 100644 index f0d7d665..00000000 --- a/bootstrap/k3s/pkg/locking/control_plane_init_mutex_test.go +++ /dev/null @@ -1,308 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package locking - -import ( - "context" - "errors" - "fmt" - "testing" - - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -const ( - clusterName = "test-cluster" - clusterNamespace = "test-namespace" -) - -var ( - ctx = ctrl.SetupSignalHandler() -) - -func TestControlPlaneInitMutex_Lock(t *testing.T) { - g := NewWithT(t) - - scheme := runtime.NewScheme() - g.Expect(clusterv1.AddToScheme(scheme)).To(Succeed()) - g.Expect(corev1.AddToScheme(scheme)).To(Succeed()) - - uid := types.UID("test-uid") - tests := []struct { - name string - client client.Client - shouldAcquire bool - }{ - { - name: "should successfully acquire lock if the config cannot be found", - client: &fakeClient{ - Client: fake.NewClientBuilder().WithScheme(scheme).Build(), - getError: apierrors.NewNotFound(schema.GroupResource{Group: "", Resource: "configmaps"}, fmt.Sprintf("%s-controlplane", uid)), - }, - shouldAcquire: true, - }, - { - name: "should not acquire lock if already exits", - client: &fakeClient{ - Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(&corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configMapName(clusterName), - Namespace: clusterNamespace, - }, - }).Build(), - }, - shouldAcquire: false, - }, - { - name: "should not acquire lock if cannot create config map", - client: &fakeClient{ - Client: fake.NewClientBuilder().WithScheme(scheme).Build(), - getError: apierrors.NewNotFound(schema.GroupResource{Group: "", Resource: "configmaps"}, configMapName(clusterName)), - createError: errors.New("create error"), - }, - shouldAcquire: false, - }, - { - name: "should not acquire lock if config map already exists while creating", - client: &fakeClient{ - Client: fake.NewClientBuilder().WithScheme(scheme).Build(), - getError: apierrors.NewNotFound(schema.GroupResource{Group: "", Resource: "configmaps"}, fmt.Sprintf("%s-controlplane", uid)), - createError: apierrors.NewAlreadyExists(schema.GroupResource{Group: "", Resource: "configmaps"}, fmt.Sprintf("%s-controlplane", uid)), - }, - shouldAcquire: false, - }, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - gs := NewWithT(t) - - l := &ControlPlaneInitMutex{ - client: tc.client, - } - - cluster := &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: clusterNamespace, - Name: clusterName, - UID: uid, - }, - } - machine := &clusterv1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("machine-%s", cluster.Name), - }, - } - - gs.Expect(l.Lock(ctx, cluster, machine)).To(Equal(tc.shouldAcquire)) - }) - } -} - -func TestControlPlaneInitMutex_LockWithMachineDeletion(t *testing.T) { - g := NewWithT(t) - - scheme := runtime.NewScheme() - g.Expect(clusterv1.AddToScheme(scheme)).To(Succeed()) - g.Expect(corev1.AddToScheme(scheme)).To(Succeed()) - - newMachineName := "new-machine" - tests := []struct { - name string - client client.Client - expectedMachineName string - }{ - { - name: "should not give the lock to new machine if the machine that created it does exist", - client: &fakeClient{ - Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects( - &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configMapName(clusterName), - Namespace: clusterNamespace}, - Data: map[string]string{ - "lock-information": "{\"machineName\":\"existent-machine\"}", - }}, - &clusterv1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: "existent-machine", - Namespace: clusterNamespace, - }, - }, - ).Build(), - }, - expectedMachineName: "existent-machine", - }, - { - name: "should give the lock to new machine if the machine that created it does not exist", - client: &fakeClient{ - Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects( - &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configMapName(clusterName), - Namespace: clusterNamespace}, - Data: map[string]string{ - "lock-information": "{\"machineName\":\"non-existent-machine\"}", - }}, - ).Build(), - }, - expectedMachineName: newMachineName, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - l := &ControlPlaneInitMutex{ - client: tc.client, - } - - cluster := &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: clusterNamespace, - Name: clusterName, - }, - } - machine := &clusterv1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: newMachineName, - }, - } - - g.Eventually(func(g Gomega) error { - l.Lock(ctx, cluster, machine) - - cm := &corev1.ConfigMap{} - g.Expect(tc.client.Get(ctx, client.ObjectKey{ - Name: configMapName(clusterName), - Namespace: cluster.Namespace, - }, cm)).To(Succeed()) - - info, err := semaphore{cm}.information() - g.Expect(err).To(BeNil()) - - g.Expect(info.MachineName).To(Equal(tc.expectedMachineName)) - return nil - }, "20s").Should(Succeed()) - }) - } -} - -func TestControlPlaneInitMutex_UnLock(t *testing.T) { - uid := types.UID("test-uid") - configMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configMapName(clusterName), - Namespace: clusterNamespace, - }, - } - tests := []struct { - name string - client client.Client - shouldRelease bool - }{ - { - name: "should release lock by deleting config map", - client: &fakeClient{ - Client: fake.NewClientBuilder().Build(), - }, - shouldRelease: true, - }, - { - name: "should not release lock if cannot delete config map", - client: &fakeClient{ - Client: fake.NewClientBuilder().WithObjects(configMap).Build(), - deleteError: errors.New("delete error"), - }, - shouldRelease: false, - }, - { - name: "should release lock if config map does not exist", - client: &fakeClient{ - Client: fake.NewClientBuilder().Build(), - getError: apierrors.NewNotFound(schema.GroupResource{Group: "", Resource: "configmaps"}, fmt.Sprintf("%s-controlplane", uid)), - }, - shouldRelease: true, - }, - { - name: "should not release lock if error while getting config map", - client: &fakeClient{ - Client: fake.NewClientBuilder().Build(), - getError: errors.New("get error"), - }, - shouldRelease: false, - }, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - gs := NewWithT(t) - - l := &ControlPlaneInitMutex{ - client: tc.client, - } - - cluster := &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: clusterNamespace, - Name: clusterName, - UID: uid, - }, - } - - gs.Expect(l.Unlock(ctx, cluster)).To(Equal(tc.shouldRelease)) - }) - } -} - -type fakeClient struct { - client.Client - getError error - createError error - deleteError error -} - -func (fc *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if fc.getError != nil { - return fc.getError - } - return fc.Client.Get(ctx, key, obj) -} - -func (fc *fakeClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { - if fc.createError != nil { - return fc.createError - } - return fc.Client.Create(ctx, obj, opts...) -} - -func (fc *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { - if fc.deleteError != nil { - return fc.deleteError - } - return fc.Client.Delete(ctx, obj, opts...) -} diff --git a/bootstrap/k3s/pkg/types/config.go b/bootstrap/k3s/pkg/types/config.go deleted file mode 100644 index f6c4f652..00000000 --- a/bootstrap/k3s/pkg/types/config.go +++ /dev/null @@ -1,131 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package types - -// DefaultK3sConfigLocation is the default location for the k3s config file. -const DefaultK3sConfigLocation = "/etc/rancher/k3s/config.yaml" - -// K3sServerConfiguration is the configuration for the k3s server. -type K3sServerConfiguration struct { - // Database - DataStoreEndPoint string `json:"datastore-endpoint,omitempty"` - DataStoreCAFile string `json:"datastore-cafile,omitempty"` - DataStoreCertFile string `json:"datastore-certfile,omitempty"` - DataStoreKeyFile string `json:"datastore-keyfile,omitempty"` - - // Cluster - Token string `json:"token,omitempty"` - TokenFile string `json:"token-file,omitempty"` - Server string `json:"server,omitempty"` - ClusterInit bool `json:"cluster-init,omitempty"` - - // Listener - // BindAddress k3s bind address. - BindAddress string `json:"bind-address,omitempty"` - // HTTPSListenPort HTTPS listen port. - HTTPSListenPort int `json:"https-listen-port,omitempty"` - // AdvertiseAddress IP address that apiserver uses to advertise to members of the cluster. - AdvertiseAddress string `json:"advertise-address,omitempty"` - // AdvertisePort Port that apiserver uses to advertise to members of the cluster (default: listen-port). - AdvertisePort int `json:"advertise-port,omitempty"` - // TLSSan Add additional hostname or IP as a Subject Alternative Name in the TLS cert. - TLSSan string `json:"tls-san,omitempty"` - - // Networking - // ClusterCIDR Network CIDR to use for pod IPs. - ClusterCIDR string `json:"cluster-cidr,omitempty"` - // ServiceCIDR Network CIDR to use for services IPs. - ServiceCIDR string `json:"service-cidr,omitempty"` - // ServiceNodePortRange Port range to reserve for services with NodePort visibility. - ServiceNodePortRange string `json:"service-node-port-range,omitempty"` - // ClusterDNS cluster IP for coredns service. Should be in your service-cidr range. - ClusterDNS string `json:"cluster-dns,omitempty"` - // ClusterDomain cluster Domain. - ClusterDomain string `json:"cluster-domain,omitempty"` - // FlannelBackend One of ‘none’, ‘vxlan’, ‘ipsec’, ‘host-gw’, or ‘wireguard’. (default: vxlan) - FlannelBackend string `json:"flannel-backend,omitempty"` - - // Kubernetes components - // Disable do not deploy packaged components and delete any deployed components - // (valid items: coredns, servicelb, traefik,local-storage, metrics-server). - Disable string `json:"disable,omitempty"` - // DisableKubeProxy disable running kube-proxy. - DisableKubeProxy bool `json:"disable-kube-roxy,omitempty"` - // DisableNetworkPolicy disable k3s default network policy controller. - DisableNetworkPolicy bool `json:"disable-network-policy,omitempty"` - // DisableHelmController disable Helm controller. - DisableHelmController bool `json:"disable-helm-controller,omitempty"` - - // Kubernetes processes - // DisableCloudController Disable k3s default cloud controller manager. - DisableCloudController bool `json:"disable-cloud-controller,omitempty"` - // KubeAPIServerArgs Customized flag for kube-apiserver process. - KubeAPIServerArgs []string `json:"kube-apiserver-arg,omitempty"` - // KubeControllerManagerArgs Customized flag for kube-controller-manager process. - KubeControllerManagerArgs []string `json:"kube-controller-manager-arg,omitempty"` - // KubeSchedulerArgs Customized flag for kube-scheduler process. - KubeSchedulerArgs []string `json:"kube-scheduler-args,omitempty"` - - // Agent - K3sAgentConfiguration `json:",inline"` -} - -// K3sAgentConfiguration is the configuration for the k3s agent. -type K3sAgentConfiguration struct { - // Cluster - Token string `json:"token,omitempty"` - TokenFile string `json:"token-file,omitempty"` - Server string `json:"server,omitempty"` - - // NodeName k3s node name. - NodeName string `json:"node-name,omitempty"` - // NodeLabels registering and starting kubelet with set of labels. - NodeLabels []string `json:"node-label,omitempty"` - // NodeTaints registering and starting kubelet with set of taints. - NodeTaints []string `json:"node-taint,omitempty"` - // SeLinux Enable SELinux in containerd - SeLinux bool `json:"selinux,omitempty"` - // LBServerPort - // Local port for supervisor client load-balancer. - // If the supervisor and apiserver are not colocated an additional port 1 less than this port - // will also be used for the apiserver client load-balancer. (default: 6444) - LBServerPort int `json:"lb-server-port,omitempty"` - // DataDir Folder to hold state. - DataDir string `json:"data-dir,omitempty"` - - // Runtime - // ContainerRuntimeEndpoint Disable embedded containerd and use alternative CRI implementation. - ContainerRuntimeEndpoint string `json:"container-runtime-endpoint,omitempty"` - // PauseImage Customized pause image for containerd or Docker sandbox. - PauseImage string `json:"pause-image,omitempty"` - // PrivateRegistry Path to a private registry configuration file. - PrivateRegistry string `json:"private-registry,omitempty"` - - // Networking - // NodeIP IP address to advertise for node. - NodeIP string `json:"node-ip,omitempty"` - // NodeExternalIP External IP address to advertise for node. - NodeExternalIP string `json:"node-external-ip,omitempty"` - // ResolvConf Path to Kubelet resolv.conf file. - ResolvConf string `json:"resolv-conf,omitempty"` - - // Kubernetes - // KubeletArgs Customized flag for kubelet process. - KubeletArgs []string `json:"kubelet-arg,omitempty"` - // KubeProxyArgs Customized flag for kube-proxy process. - KubeProxyArgs []string `json:"kube-proxy-arg,omitempty"` -} diff --git a/bootstrap/k3s/pkg/types/doc.go b/bootstrap/k3s/pkg/types/doc.go deleted file mode 100644 index 7a55aa89..00000000 --- a/bootstrap/k3s/pkg/types/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package types contains k3s config types. -package types diff --git a/bootstrap/k3s/pkg/types/util.go b/bootstrap/k3s/pkg/types/util.go deleted file mode 100644 index f5791044..00000000 --- a/bootstrap/k3s/pkg/types/util.go +++ /dev/null @@ -1,177 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package types - -import ( - "fmt" - "strings" - - "github.com/jinzhu/copier" - kubeyaml "sigs.k8s.io/yaml" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" -) - -// MarshalInitServerConfiguration marshals the ServerConfiguration object into a string. -func MarshalInitServerConfiguration(spec *infrabootstrapv1.K3sConfigSpec, token string) (string, error) { - obj := spec.ServerConfiguration - serverConfig := &K3sServerConfiguration{} - if err := copier.Copy(serverConfig, obj.Database); err != nil { - return "", err - } - if err := copier.Copy(serverConfig, obj.Listener); err != nil { - return "", err - } - if err := copier.Copy(serverConfig, obj.Networking); err != nil { - return "", err - } - if err := copier.Copy(serverConfig, obj.KubernetesComponents); err != nil { - return "", err - } - - serverConfig.Token = token - serverConfig.ClusterInit = *obj.Database.ClusterInit - - serverConfig.DisableCloudController = true - serverConfig.KubeAPIServerArgs = append(obj.KubernetesProcesses.KubeAPIServerArgs, "anonymous-auth=true", getTLSCipherSuiteArg()) - serverConfig.KubeControllerManagerArgs = append(obj.KubernetesProcesses.KubeControllerManagerArgs, "cloud-provider=external") - serverConfig.KubeSchedulerArgs = obj.KubernetesProcesses.KubeSchedulerArgs - - serverConfig.K3sAgentConfiguration = K3sAgentConfiguration{ - NodeName: obj.Agent.Node.NodeName, - NodeLabels: obj.Agent.Node.NodeLabels, - NodeTaints: obj.Agent.Node.NodeTaints, - SeLinux: obj.Agent.Node.SeLinux, - LBServerPort: obj.Agent.Node.LBServerPort, - DataDir: obj.Agent.Node.DataDir, - ContainerRuntimeEndpoint: obj.Agent.Runtime.ContainerRuntimeEndpoint, - PauseImage: obj.Agent.Runtime.PauseImage, - PrivateRegistry: obj.Agent.Runtime.PrivateRegistry, - NodeIP: obj.Agent.Networking.NodeIP, - NodeExternalIP: obj.Agent.Networking.NodeExternalIP, - ResolvConf: obj.Agent.Networking.ResolvConf, - KubeletArgs: obj.Agent.KubernetesAgentProcesses.KubeletArgs, - KubeProxyArgs: obj.Agent.KubernetesAgentProcesses.KubeProxyArgs, - } - - b, err := kubeyaml.Marshal(serverConfig) - if err != nil { - return "", err - } - return string(b), nil -} - -// MarshalJoinServerConfiguration marshals the join ServerConfiguration object into a string. -func MarshalJoinServerConfiguration(spec *infrabootstrapv1.K3sConfigSpec) (string, error) { - obj := spec.ServerConfiguration - serverConfig := &K3sServerConfiguration{} - if err := copier.Copy(serverConfig, obj.Database); err != nil { - return "", err - } - if err := copier.Copy(serverConfig, obj.Listener); err != nil { - return "", err - } - if err := copier.Copy(serverConfig, obj.Networking); err != nil { - return "", err - } - if err := copier.Copy(serverConfig, obj.KubernetesComponents); err != nil { - return "", err - } - - serverConfig.TokenFile = spec.Cluster.TokenFile - serverConfig.Token = spec.Cluster.Token - serverConfig.Server = spec.Cluster.Server - - serverConfig.DisableCloudController = true - serverConfig.KubeAPIServerArgs = append(obj.KubernetesProcesses.KubeAPIServerArgs, "anonymous-auth=true", getTLSCipherSuiteArg()) - serverConfig.KubeControllerManagerArgs = append(obj.KubernetesProcesses.KubeControllerManagerArgs, "cloud-provider=external") - serverConfig.KubeSchedulerArgs = obj.KubernetesProcesses.KubeSchedulerArgs - - serverConfig.K3sAgentConfiguration = K3sAgentConfiguration{ - NodeName: obj.Agent.Node.NodeName, - NodeLabels: obj.Agent.Node.NodeLabels, - NodeTaints: obj.Agent.Node.NodeTaints, - SeLinux: obj.Agent.Node.SeLinux, - LBServerPort: obj.Agent.Node.LBServerPort, - DataDir: obj.Agent.Node.DataDir, - ContainerRuntimeEndpoint: obj.Agent.Runtime.ContainerRuntimeEndpoint, - PauseImage: obj.Agent.Runtime.PauseImage, - PrivateRegistry: obj.Agent.Runtime.PrivateRegistry, - NodeIP: obj.Agent.Networking.NodeIP, - NodeExternalIP: obj.Agent.Networking.NodeExternalIP, - ResolvConf: obj.Agent.Networking.ResolvConf, - KubeletArgs: obj.Agent.KubernetesAgentProcesses.KubeletArgs, - KubeProxyArgs: obj.Agent.KubernetesAgentProcesses.KubeProxyArgs, - } - - b, err := kubeyaml.Marshal(serverConfig) - if err != nil { - return "", err - } - return string(b), nil -} - -// MarshalJoinAgentConfiguration marshals the join AgentConfiguration object into a string. -func MarshalJoinAgentConfiguration(spec *infrabootstrapv1.K3sConfigSpec) (string, error) { - obj := spec.AgentConfiguration - agentConfig := &K3sAgentConfiguration{} - if err := copier.Copy(agentConfig, obj.Node); err != nil { - return "", err - } - if err := copier.Copy(agentConfig, obj.Networking); err != nil { - return "", err - } - if err := copier.Copy(agentConfig, obj.Runtime); err != nil { - return "", err - } - if err := copier.Copy(agentConfig, obj.KubernetesAgentProcesses); err != nil { - return "", err - } - - agentConfig.TokenFile = spec.Cluster.TokenFile - agentConfig.Token = spec.Cluster.Token - agentConfig.Server = spec.Cluster.Server - - b, err := kubeyaml.Marshal(agentConfig) - if err != nil { - return "", err - } - return string(b), nil -} - -func getTLSCipherSuiteArg() string { - ciphers := []string{ - // Modern Compatibility recommended configuration in - // https://wiki.mozilla.org/Security/Server_Side_TLS#Modern_compatibility - "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", - "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", - "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", - "TLS_RSA_WITH_AES_128_GCM_SHA256", - "TLS_RSA_WITH_AES_256_GCM_SHA384", - } - - ciphersList := "" - for _, cc := range ciphers { - ciphersList += cc + "," - } - ciphersList = strings.TrimRight(ciphersList, ",") - - return fmt.Sprintf("tls-cipher-suites=%s", ciphersList) -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/addons_types.go b/cmd/kk/apis/kubekey/v1alpha2/addons_types.go deleted file mode 100644 index 5500317b..00000000 --- a/cmd/kk/apis/kubekey/v1alpha2/addons_types.go +++ /dev/null @@ -1,44 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1alpha2 - -type Addon struct { - Name string `yaml:"name" json:"name,omitempty"` - Namespace string `yaml:"namespace" json:"namespace,omitempty"` - Sources Sources `yaml:"sources" json:"sources,omitempty"` - Retries int `yaml:"retries" json:"retries,omitempty"` - Delay int `yaml:"delay" json:"delay,omitempty"` -} - -type Sources struct { - Chart Chart `yaml:"chart" json:"chart,omitempty"` - Yaml Yaml `yaml:"yaml" json:"yaml,omitempty"` -} - -type Chart struct { - Name string `yaml:"name" json:"name,omitempty"` - Repo string `yaml:"repo" json:"repo,omitempty"` - Path string `yaml:"path" json:"path,omitempty"` - Version string `yaml:"version" json:"version,omitempty"` - ValuesFile string `yaml:"valuesFile" json:"valuesFile,omitempty"` - Values []string `yaml:"values" json:"values,omitempty"` - Wait bool `yaml:"wait" json:"wait,omitempty"` -} - -type Yaml struct { - Path []string `yaml:"path" json:"path,omitempty"` -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go b/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go deleted file mode 100644 index 1bd4f779..00000000 --- a/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go +++ /dev/null @@ -1,305 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1alpha2 - -import ( - "fmt" - "regexp" - "strconv" - "strings" - - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -// ClusterSpec defines the desired state of Cluster -type ClusterSpec struct { - Hosts []HostCfg `yaml:"hosts" json:"hosts,omitempty"` - RoleGroups map[string][]string `yaml:"roleGroups" json:"roleGroups,omitempty"` - ControlPlaneEndpoint ControlPlaneEndpoint `yaml:"controlPlaneEndpoint" json:"controlPlaneEndpoint,omitempty"` - System System `yaml:"system" json:"system,omitempty"` - Etcd EtcdCluster `yaml:"etcd" json:"etcd,omitempty"` - DNS DNS `yaml:"dns" json:"dns,omitempty"` - Kubernetes Kubernetes `yaml:"kubernetes" json:"kubernetes,omitempty"` - Network NetworkConfig `yaml:"network" json:"network,omitempty"` - Storage StorageConfig `yaml:"storage" json:"storage,omitempty"` - Registry RegistryConfig `yaml:"registry" json:"registry,omitempty"` - Addons []Addon `yaml:"addons" json:"addons,omitempty"` - KubeSphere KubeSphere `json:"kubesphere,omitempty"` -} - -type Cluster struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec ClusterSpec `json:"spec,omitempty"` -} - -// HostCfg defines host information for cluster. -type HostCfg struct { - Name string `yaml:"name,omitempty" json:"name,omitempty"` - Address string `yaml:"address,omitempty" json:"address,omitempty"` - InternalAddress string `yaml:"internalAddress,omitempty" json:"internalAddress,omitempty"` - Port int `yaml:"port,omitempty" json:"port,omitempty"` - User string `yaml:"user,omitempty" json:"user,omitempty"` - Password string `yaml:"password,omitempty" json:"password,omitempty"` - PrivateKey string `yaml:"privateKey,omitempty" json:"privateKey,omitempty"` - PrivateKeyPath string `yaml:"privateKeyPath,omitempty" json:"privateKeyPath,omitempty"` - Arch string `yaml:"arch,omitempty" json:"arch,omitempty"` - Timeout *int64 `yaml:"timeout,omitempty" json:"timeout,omitempty"` - - // Labels defines the kubernetes labels for the node. - Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty"` -} - -// ControlPlaneEndpoint defines the control plane endpoint information for cluster. -type ControlPlaneEndpoint struct { - InternalLoadbalancer string `yaml:"internalLoadbalancer" json:"internalLoadbalancer,omitempty"` - Domain string `yaml:"domain" json:"domain,omitempty"` - ExternalDNS *bool `yaml:"externalDNS" json:"externalDNS"` - Address string `yaml:"address" json:"address,omitempty"` - Port int `yaml:"port" json:"port,omitempty"` - KubeVip KubeVip `yaml:"kubevip" json:"kubevip,omitempty"` -} - -type KubeVip struct { - Mode string `yaml:"mode" json:"mode,omitempty"` -} - -// CustomScripts defines the custom shell scripts for each node to exec before and finished kubernetes install. -type CustomScripts struct { - Name string `yaml:"name" json:"name,omitempty"` - Bash string `yaml:"bash" json:"bash,omitempty"` - Materials []string `yaml:"materials" json:"materials,omitempty"` -} - -// System defines the system config for each node in cluster. -type System struct { - NtpServers []string `yaml:"ntpServers" json:"ntpServers,omitempty"` - Timezone string `yaml:"timezone" json:"timezone,omitempty"` - Rpms []string `yaml:"rpms" json:"rpms,omitempty"` - Debs []string `yaml:"debs" json:"debs,omitempty"` - PreInstall []CustomScripts `yaml:"preInstall" json:"preInstall,omitempty"` - PostInstall []CustomScripts `yaml:"postInstall" json:"postInstall,omitempty"` - SkipConfigureOS bool `yaml:"skipConfigureOS" json:"skipConfigureOS,omitempty"` -} - -// RegistryConfig defines the configuration information of the image's repository. -type RegistryConfig struct { - Type string `yaml:"type" json:"type,omitempty"` - RegistryMirrors []string `yaml:"registryMirrors" json:"registryMirrors,omitempty"` - InsecureRegistries []string `yaml:"insecureRegistries" json:"insecureRegistries,omitempty"` - PrivateRegistry string `yaml:"privateRegistry" json:"privateRegistry,omitempty"` - DataRoot string `yaml:"dataRoot" json:"dataRoot,omitempty"` - NamespaceOverride string `yaml:"namespaceOverride" json:"namespaceOverride,omitempty"` - BridgeIP string `yaml:"bridgeIP" json:"bridgeIP,omitempty"` - Auths runtime.RawExtension `yaml:"auths" json:"auths,omitempty"` -} - -// KubeSphere defines the configuration information of the KubeSphere. -type KubeSphere struct { - Enabled bool `json:"enabled,omitempty"` - Version string `json:"version,omitempty"` - Configurations string `json:"configurations,omitempty"` -} - -// GenerateCertSANs is used to generate cert sans for cluster. -func (cfg *ClusterSpec) GenerateCertSANs() []string { - clusterSvc := fmt.Sprintf("kubernetes.default.svc.%s", cfg.Kubernetes.DNSDomain) - defaultCertSANs := []string{"kubernetes", "kubernetes.default", "kubernetes.default.svc", clusterSvc, "localhost", "127.0.0.1"} - extraCertSANs := make([]string, 0) - - extraCertSANs = append(extraCertSANs, cfg.ControlPlaneEndpoint.Domain) - - if cfg.ControlPlaneEndpoint.Address != "" { - extraCertSANs = append(extraCertSANs, cfg.ControlPlaneEndpoint.Address) - } - - for _, host := range cfg.Hosts { - extraCertSANs = append(extraCertSANs, host.Name) - extraCertSANs = append(extraCertSANs, fmt.Sprintf("%s.%s", host.Name, cfg.Kubernetes.DNSDomain)) - if host.Address != cfg.ControlPlaneEndpoint.Address { - extraCertSANs = append(extraCertSANs, host.Address) - } - if host.InternalAddress != host.Address && host.InternalAddress != cfg.ControlPlaneEndpoint.Address { - extraCertSANs = append(extraCertSANs, host.InternalAddress) - } - } - - extraCertSANs = append(extraCertSANs, util.ParseIp(cfg.Network.KubeServiceCIDR)[0]) - - defaultCertSANs = append(defaultCertSANs, extraCertSANs...) - - if cfg.Kubernetes.ApiserverCertExtraSans != nil { - defaultCertSANs = append(defaultCertSANs, cfg.Kubernetes.ApiserverCertExtraSans...) - } - - return defaultCertSANs -} - -// GroupHosts is used to group hosts according to the configuration file.s -func (cfg *ClusterSpec) GroupHosts() map[string][]*KubeHost { - hostMap := make(map[string]*KubeHost) - for _, hostCfg := range cfg.Hosts { - host := toHosts(hostCfg) - hostMap[host.Name] = host - } - - roleGroups := cfg.ParseRolesList(hostMap) - - //Check that the parameters under roleGroups are incorrect - if len(roleGroups[Master]) == 0 && len(roleGroups[ControlPlane]) == 0 { - logger.Log.Fatal(errors.New("The number of master/control-plane cannot be 0")) - } - if len(roleGroups[Etcd]) == 0 && cfg.Etcd.Type == KubeKey { - logger.Log.Fatal(errors.New("The number of etcd cannot be 0")) - } - - for _, host := range roleGroups[ControlPlane] { - host.SetRole(Master) - roleGroups[Master] = append(roleGroups[Master], host) - } - - return roleGroups -} - -// +kubebuilder:object:generate=false -type KubeHost struct { - *connector.BaseHost - Labels map[string]string -} - -func toHosts(cfg HostCfg) *KubeHost { - host := connector.NewHost() - host.Name = cfg.Name - host.Address = cfg.Address - host.InternalAddress = cfg.InternalAddress - host.Port = cfg.Port - host.User = cfg.User - host.Password = cfg.Password - host.PrivateKey = cfg.PrivateKey - host.PrivateKeyPath = cfg.PrivateKeyPath - host.Arch = cfg.Arch - host.Timeout = *cfg.Timeout - - kubeHost := &KubeHost{ - BaseHost: host, - Labels: cfg.Labels, - } - return kubeHost -} - -// ClusterIP is used to get the kube-apiserver service address inside the cluster. -func (cfg *ClusterSpec) ClusterIP() string { - return util.ParseIp(cfg.Network.KubeServiceCIDR)[0] -} - -// CorednsClusterIP is used to get the coredns service address inside the cluster. -func (cfg *ClusterSpec) CorednsClusterIP() string { - return util.ParseIp(cfg.Network.KubeServiceCIDR)[2] -} - -// ClusterDNS is used to get the dns server address inside the cluster. -func (cfg *ClusterSpec) ClusterDNS() string { - if cfg.Kubernetes.EnableNodelocaldns() { - return "169.254.25.10" - } else { - return cfg.CorednsClusterIP() - } -} - -// ParseRolesList is used to parse the host grouping list. -func (cfg *ClusterSpec) ParseRolesList(hostMap map[string]*KubeHost) map[string][]*KubeHost { - roleGroupLists := make(map[string][]*KubeHost) - for role, hosts := range cfg.RoleGroups { - roleGroup := make([]string, 0) - for _, host := range hosts { - h := make([]string, 0) - if strings.Contains(host, "[") && strings.Contains(host, "]") && strings.Contains(host, ":") { - rangeHosts := getHostsRange(host, hostMap, role) - h = append(h, rangeHosts...) - } else { - if err := hostVerify(hostMap, host, role); err != nil { - logger.Log.Fatal(err) - } - h = append(h, host) - } - - roleGroup = append(roleGroup, h...) - for _, hostName := range h { - if h, ok := hostMap[hostName]; ok { - roleGroupAppend(roleGroupLists, role, h) - } else { - logger.Log.Fatal(fmt.Errorf("incorrect nodeName under roleGroups/%s in the configuration file", role)) - } - } - } - } - - return roleGroupLists -} - -func roleGroupAppend(roleGroupLists map[string][]*KubeHost, role string, host *KubeHost) { - host.SetRole(role) - r := roleGroupLists[role] - r = append(r, host) - roleGroupLists[role] = r -} - -func getHostsRange(rangeStr string, hostMap map[string]*KubeHost, group string) []string { - hostRangeList := make([]string, 0) - r := regexp.MustCompile(`\[(\d+)\:(\d+)\]`) - nameSuffix := r.FindStringSubmatch(rangeStr) - namePrefix := strings.Split(rangeStr, nameSuffix[0])[0] - nameSuffixStart, _ := strconv.Atoi(nameSuffix[1]) - nameSuffixEnd, _ := strconv.Atoi(nameSuffix[2]) - for i := nameSuffixStart; i <= nameSuffixEnd; i++ { - if err := hostVerify(hostMap, fmt.Sprintf("%s%d", namePrefix, i), group); err != nil { - logger.Log.Fatal(err) - } - hostRangeList = append(hostRangeList, fmt.Sprintf("%s%d", namePrefix, i)) - } - return hostRangeList -} - -func hostVerify(hostMap map[string]*KubeHost, hostName string, group string) error { - if _, ok := hostMap[hostName]; !ok { - return fmt.Errorf("[%s] is in [%s] group, but not in hosts list", hostName, group) - } - return nil -} - -func (c ControlPlaneEndpoint) IsInternalLBEnabled() bool { - return c.InternalLoadbalancer == Haproxy -} - -func (c ControlPlaneEndpoint) IsInternalLBEnabledVip() bool { - return c.InternalLoadbalancer == Kubevip -} - -// EnableExternalDNS is used to determine whether to use external dns to resolve kube-apiserver domain. -func (c *ControlPlaneEndpoint) EnableExternalDNS() bool { - if c.ExternalDNS == nil { - return false - } - return *c.ExternalDNS -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/default.go b/cmd/kk/apis/kubekey/v1alpha2/default.go deleted file mode 100644 index 64701675..00000000 --- a/cmd/kk/apis/kubekey/v1alpha2/default.go +++ /dev/null @@ -1,352 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1alpha2 - -import ( - "fmt" - "os" - "strings" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -const ( - DefaultPreDir = "kubekey" - DefaultTmpDir = "/tmp/kubekey" - DefaultSSHPort = 22 - DefaultLBPort = 6443 - DefaultApiserverPort = 6443 - DefaultLBDomain = "lb.kubesphere.local" - DefaultNetworkPlugin = "calico" - DefaultPodsCIDR = "10.233.64.0/18" - DefaultServiceCIDR = "10.233.0.0/18" - DefaultKubeImageNamespace = "kubesphere" - DefaultClusterName = "cluster.local" - DefaultDNSDomain = "cluster.local" - DefaultArch = "amd64" - DefaultSSHTimeout = 30 - DefaultEtcdVersion = "v3.5.6" - DefaultEtcdPort = "2379" - DefaultDockerVersion = "24.0.6" - DefaultContainerdVersion = "1.7.8" - DefaultRuncVersion = "v1.1.10" - DefaultCrictlVersion = "v1.24.0" - DefaultKubeVersion = "v1.23.10" - DefaultCalicoVersion = "v3.26.1" - DefaultFlannelVersion = "v0.21.3" - DefaultFlannelCniPluginVersion = "v1.1.2" - DefaultCniVersion = "v1.2.0" - DefaultCiliumVersion = "v1.11.7" - DefaulthybridnetVersion = "v0.8.6" - DefaultKubeovnVersion = "v1.10.6" - DefalutMultusVersion = "v3.8" - DefaultHelmVersion = "v3.9.0" - DefaultDockerComposeVersion = "v2.2.2" - DefaultRegistryVersion = "2" - DefaultHarborVersion = "v2.5.3" - DefaultMaxPods = 110 - DefaultPodPidsLimit = 10000 - DefaultNodeCidrMaskSize = 24 - DefaultIPIPMode = "Always" - DefaultVXLANMode = "Never" - DefaultVethMTU = 0 - DefaultBackendMode = "vxlan" - DefaultProxyMode = "ipvs" - DefaultCrioEndpoint = "unix:///var/run/crio/crio.sock" - DefaultContainerdEndpoint = "unix:///run/containerd/containerd.sock" - DefaultIsulaEndpoint = "unix:///var/run/isulad.sock" - Etcd = "etcd" - Master = "master" - ControlPlane = "control-plane" - Worker = "worker" - K8s = "k8s" - Registry = "registry" - DefaultEtcdBackupDir = "/var/backups/kube_etcd" - DefaultEtcdBackupPeriod = 1440 - DefaultKeepBackNumber = 5 - DefaultEtcdBackupScriptDir = "/usr/local/bin/kube-scripts" - DefaultPodGateway = "10.233.64.1" - DefaultJoinCIDR = "100.64.0.0/16" - DefaultNetworkType = "geneve" - DefaultTunnelType = "geneve" - DefaultPodNicType = "veth-pair" - DefaultModules = "kube_ovn_fastpath.ko" - DefaultRPMs = "openvswitch-kmod" - DefaultVlanID = "100" - DefaultOvnLabel = "node-role.kubernetes.io/control-plane" - DefaultDPDKVersion = "19.11" - DefaultDNSAddress = "114.114.114.114" - DefaultDpdkTunnelIface = "br-phy" - DefaultCNIConfigPriority = "01" - DefaultOpenEBSBasePath = "/var/openebs/local" - - Docker = "docker" - Containerd = "containerd" - Crio = "crio" - Isula = "isula" - - Haproxy = "haproxy" - Kubevip = "kube-vip" - DefaultKubeVipMode = "ARP" -) - -func (cfg *ClusterSpec) SetDefaultClusterSpec() (*ClusterSpec, map[string][]*KubeHost) { - clusterCfg := ClusterSpec{} - - clusterCfg.Hosts = SetDefaultHostsCfg(cfg) - clusterCfg.RoleGroups = cfg.RoleGroups - clusterCfg.Etcd = SetDefaultEtcdCfg(cfg) - roleGroups := clusterCfg.GroupHosts() - clusterCfg.ControlPlaneEndpoint = SetDefaultLBCfg(cfg, roleGroups[Master]) - clusterCfg.Network = SetDefaultNetworkCfg(cfg) - clusterCfg.Storage = SetDefaultStorageCfg(cfg) - clusterCfg.System = cfg.System - clusterCfg.Kubernetes = SetDefaultClusterCfg(cfg) - clusterCfg.DNS = cfg.DNS - clusterCfg.Registry = cfg.Registry - clusterCfg.Addons = cfg.Addons - clusterCfg.KubeSphere = cfg.KubeSphere - - if cfg.Kubernetes.ClusterName == "" { - clusterCfg.Kubernetes.ClusterName = DefaultClusterName - } - if cfg.Kubernetes.Version == "" { - clusterCfg.Kubernetes.Version = DefaultKubeVersion - } - if cfg.Kubernetes.MaxPods == 0 { - clusterCfg.Kubernetes.MaxPods = DefaultMaxPods - } - if cfg.Kubernetes.PodPidsLimit == 0 { - clusterCfg.Kubernetes.PodPidsLimit = DefaultPodPidsLimit - } - if cfg.Kubernetes.NodeCidrMaskSize == 0 { - clusterCfg.Kubernetes.NodeCidrMaskSize = DefaultNodeCidrMaskSize - } - if cfg.Kubernetes.ProxyMode == "" { - clusterCfg.Kubernetes.ProxyMode = DefaultProxyMode - } - return &clusterCfg, roleGroups -} - -func SetDefaultHostsCfg(cfg *ClusterSpec) []HostCfg { - var hostCfg []HostCfg - if len(cfg.Hosts) == 0 { - return nil - } - for _, host := range cfg.Hosts { - if len(host.Address) == 0 && len(host.InternalAddress) > 0 { - host.Address = host.InternalAddress - } - if len(host.InternalAddress) == 0 && len(host.Address) > 0 { - host.InternalAddress = host.Address - } - if host.User == "" { - host.User = "root" - } - if host.Port == 0 { - host.Port = DefaultSSHPort - } - if host.PrivateKey == "" { - if host.Password == "" && host.PrivateKeyPath == "" { - host.PrivateKeyPath = "~/.ssh/id_rsa" - } - if host.PrivateKeyPath != "" && strings.HasPrefix(strings.TrimSpace(host.PrivateKeyPath), "~/") { - homeDir, _ := util.Home() - host.PrivateKeyPath = strings.Replace(host.PrivateKeyPath, "~/", fmt.Sprintf("%s/", homeDir), 1) - } - } - - if host.Arch == "" { - host.Arch = DefaultArch - } - - if host.Timeout == nil { - var timeout int64 - timeout = DefaultSSHTimeout - host.Timeout = &timeout - } - - hostCfg = append(hostCfg, host) - } - return hostCfg -} - -func SetDefaultLBCfg(cfg *ClusterSpec, masterGroup []*KubeHost) ControlPlaneEndpoint { - //Check whether LB should be configured - if len(masterGroup) >= 2 && !cfg.ControlPlaneEndpoint.IsInternalLBEnabled() && cfg.ControlPlaneEndpoint.Address == "" && !cfg.ControlPlaneEndpoint.EnableExternalDNS() { - fmt.Println() - fmt.Println("Warning: When there are at least two nodes in the control-plane, you should set the value of the LB address or enable the internal loadbalancer, or set 'controlPlaneEndpoint.externalDNS' to 'true' if the 'controlPlaneEndpoint.domain' can be resolved in your dns server.") - fmt.Println() - } - - // Check whether LB address and the internal LB are both enabled - if cfg.ControlPlaneEndpoint.IsInternalLBEnabled() && cfg.ControlPlaneEndpoint.Address != "" { - fmt.Println("You cannot set up the internal load balancer and the LB address at the same time.") - os.Exit(0) - } - - if (cfg.ControlPlaneEndpoint.Address == "" && !cfg.ControlPlaneEndpoint.EnableExternalDNS()) || cfg.ControlPlaneEndpoint.Address == "127.0.0.1" { - cfg.ControlPlaneEndpoint.Address = masterGroup[0].InternalAddress - } - if cfg.ControlPlaneEndpoint.Domain == "" { - cfg.ControlPlaneEndpoint.Domain = DefaultLBDomain - } - if cfg.ControlPlaneEndpoint.Port == 0 { - cfg.ControlPlaneEndpoint.Port = DefaultLBPort - } - if cfg.ControlPlaneEndpoint.KubeVip.Mode == "" { - cfg.ControlPlaneEndpoint.KubeVip.Mode = DefaultKubeVipMode - } - defaultLbCfg := cfg.ControlPlaneEndpoint - return defaultLbCfg -} - -func SetDefaultNetworkCfg(cfg *ClusterSpec) NetworkConfig { - if cfg.Network.Plugin == "" { - cfg.Network.Plugin = DefaultNetworkPlugin - } - if cfg.Network.KubePodsCIDR == "" { - cfg.Network.KubePodsCIDR = DefaultPodsCIDR - } - if cfg.Network.KubeServiceCIDR == "" { - cfg.Network.KubeServiceCIDR = DefaultServiceCIDR - } - if cfg.Network.Calico.IPIPMode == "" { - cfg.Network.Calico.IPIPMode = DefaultIPIPMode - } - if cfg.Network.Calico.VXLANMode == "" { - cfg.Network.Calico.VXLANMode = DefaultVXLANMode - } - if cfg.Network.Calico.VethMTU == 0 { - cfg.Network.Calico.VethMTU = DefaultVethMTU - } - if cfg.Network.Flannel.BackendMode == "" { - cfg.Network.Flannel.BackendMode = DefaultBackendMode - } - // kube-ovn default config - if cfg.Network.Kubeovn.KubeOvnController.PodGateway == "" { - cfg.Network.Kubeovn.KubeOvnController.PodGateway = DefaultPodGateway - } - if cfg.Network.Kubeovn.JoinCIDR == "" { - cfg.Network.Kubeovn.JoinCIDR = DefaultJoinCIDR - } - if cfg.Network.Kubeovn.Label == "" { - cfg.Network.Kubeovn.Label = DefaultOvnLabel - } - if cfg.Network.Kubeovn.KubeOvnController.VlanID == "" { - cfg.Network.Kubeovn.KubeOvnController.VlanID = DefaultVlanID - } - if cfg.Network.Kubeovn.KubeOvnController.NetworkType == "" { - cfg.Network.Kubeovn.KubeOvnController.NetworkType = DefaultNetworkType - } - if cfg.Network.Kubeovn.TunnelType == "" { - cfg.Network.Kubeovn.TunnelType = DefaultTunnelType - } - if cfg.Network.Kubeovn.KubeOvnController.PodNicType == "" { - cfg.Network.Kubeovn.KubeOvnController.PodNicType = DefaultPodNicType - } - if cfg.Network.Kubeovn.KubeOvnCni.Modules == "" { - cfg.Network.Kubeovn.KubeOvnCni.Modules = DefaultModules - } - if cfg.Network.Kubeovn.KubeOvnCni.RPMs == "" { - cfg.Network.Kubeovn.KubeOvnCni.RPMs = DefaultRPMs - } - if cfg.Network.Kubeovn.KubeOvnPinger.PingerExternalAddress == "" { - cfg.Network.Kubeovn.KubeOvnPinger.PingerExternalAddress = DefaultDNSAddress - } - if cfg.Network.Kubeovn.Dpdk.DpdkVersion == "" { - cfg.Network.Kubeovn.Dpdk.DpdkVersion = DefaultDPDKVersion - } - if cfg.Network.Kubeovn.Dpdk.DpdkTunnelIface == "" { - cfg.Network.Kubeovn.Dpdk.DpdkTunnelIface = DefaultDpdkTunnelIface - } - if cfg.Network.Kubeovn.KubeOvnCni.CNIConfigPriority == "" { - cfg.Network.Kubeovn.KubeOvnCni.CNIConfigPriority = DefaultCNIConfigPriority - } - defaultNetworkCfg := cfg.Network - - return defaultNetworkCfg -} - -func SetDefaultStorageCfg(cfg *ClusterSpec) StorageConfig { - if cfg.Storage.OpenEBS.BasePath == "" { - cfg.Storage.OpenEBS.BasePath = DefaultOpenEBSBasePath - } - defaultStorageCfg := cfg.Storage - return defaultStorageCfg -} - -func SetDefaultClusterCfg(cfg *ClusterSpec) Kubernetes { - if cfg.Kubernetes.Version == "" { - cfg.Kubernetes.Version = DefaultKubeVersion - } else { - s := strings.Split(cfg.Kubernetes.Version, "-") - if len(s) > 1 { - cfg.Kubernetes.Version = s[0] - cfg.Kubernetes.Type = s[1] - } - } - if cfg.Kubernetes.Type == "" { - cfg.Kubernetes.Type = "kubernetes" - } - if cfg.Kubernetes.ClusterName == "" { - cfg.Kubernetes.ClusterName = DefaultClusterName - } - if cfg.Kubernetes.DNSDomain == "" { - cfg.Kubernetes.DNSDomain = DefaultDNSDomain - } - if cfg.Kubernetes.ContainerManager == "" { - cfg.Kubernetes.ContainerManager = Docker - } - if cfg.Kubernetes.ContainerRuntimeEndpoint == "" { - switch cfg.Kubernetes.ContainerManager { - case Docker: - cfg.Kubernetes.ContainerRuntimeEndpoint = "" - case Crio: - cfg.Kubernetes.ContainerRuntimeEndpoint = DefaultCrioEndpoint - case Containerd: - cfg.Kubernetes.ContainerRuntimeEndpoint = DefaultContainerdEndpoint - case Isula: - cfg.Kubernetes.ContainerRuntimeEndpoint = DefaultIsulaEndpoint - default: - cfg.Kubernetes.ContainerRuntimeEndpoint = "" - } - } - defaultClusterCfg := cfg.Kubernetes - - return defaultClusterCfg -} - -func SetDefaultEtcdCfg(cfg *ClusterSpec) EtcdCluster { - if cfg.Etcd.Type == "" || ((cfg.Kubernetes.Type == "k3s" || (len(strings.Split(cfg.Kubernetes.Version, "-")) > 1) && strings.Split(cfg.Kubernetes.Version, "-")[1] == "k3s") && cfg.Etcd.Type == Kubeadm) { - cfg.Etcd.Type = KubeKey - } - if cfg.Etcd.BackupDir == "" { - cfg.Etcd.BackupDir = DefaultEtcdBackupDir - } - if cfg.Etcd.BackupPeriod == 0 { - cfg.Etcd.BackupPeriod = DefaultEtcdBackupPeriod - } - if cfg.Etcd.KeepBackupNumber == 0 { - cfg.Etcd.KeepBackupNumber = DefaultKeepBackNumber - } - if cfg.Etcd.BackupScriptDir == "" { - cfg.Etcd.BackupScriptDir = DefaultEtcdBackupScriptDir - } - - return cfg.Etcd -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/dns_types.go b/cmd/kk/apis/kubekey/v1alpha2/dns_types.go deleted file mode 100644 index aeaf0081..00000000 --- a/cmd/kk/apis/kubekey/v1alpha2/dns_types.go +++ /dev/null @@ -1,41 +0,0 @@ -/* - Copyright 2023 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1alpha2 - -type DNS struct { - DNSEtcHosts string `yaml:"dnsEtcHosts" json:"dnsEtcHosts"` - CoreDNS CoreDNS `yaml:"coredns" json:"coredns"` - NodeLocalDNS NodeLocalDNS `yaml:"nodelocaldns" json:"nodelocaldns"` -} - -type CoreDNS struct { - AdditionalConfigs string `yaml:"additionalConfigs" json:"additionalConfigs"` - ExternalZones []ExternalZone `yaml:"externalZones" json:"externalZones"` - RewriteBlock string `yaml:"rewriteBlock" json:"rewriteBlock"` - UpstreamDNSServers []string `yaml:"upstreamDNSServers" json:"upstreamDNSServers"` -} - -type NodeLocalDNS struct { - ExternalZones []ExternalZone `yaml:"externalZones" json:"externalZones"` -} - -type ExternalZone struct { - Zones []string `yaml:"zones" json:"zones"` - Nameservers []string `yaml:"nameservers" json:"nameservers"` - Cache int `yaml:"cache" json:"cache"` - Rewrite []string `yaml:"rewrite" json:"rewrite"` -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/etcd_types.go b/cmd/kk/apis/kubekey/v1alpha2/etcd_types.go deleted file mode 100644 index 6dbd3050..00000000 --- a/cmd/kk/apis/kubekey/v1alpha2/etcd_types.go +++ /dev/null @@ -1,59 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1alpha2 - -const ( - KubeKey = "kubekey" - Kubeadm = "kubeadm" - External = "external" -) - -type EtcdCluster struct { - // Type of etcd cluster, can be set to 'kubekey' 'kubeadm' 'external' - Type string `yaml:"type" json:"type,omitempty"` - // ExternalEtcd describes how to connect to an external etcd cluster when type is set to external - External ExternalEtcd `yaml:"external" json:"external,omitempty"` - BackupDir string `yaml:"backupDir" json:"backupDir,omitempty"` - BackupPeriod int `yaml:"backupPeriod" json:"backupPeriod,omitempty"` - KeepBackupNumber int `yaml:"keepBackupNumber" json:"keepBackupNumber,omitempty"` - BackupScriptDir string `yaml:"backupScript" json:"backupScript,omitempty"` - DataDir *string `yaml:"dataDir" json:"dataDir,omitempty"` - HeartbeatInterval *int `yaml:"heartbeatInterval" json:"heartbeatInterval,omitempty"` - ElectionTimeout *int `yaml:"electionTimeout" json:"electionTimeout,omitempty"` - SnapshotCount *int `yaml:"snapshotCount" json:"snapshotCount,omitempty"` - AutoCompactionRetention *int `yaml:"autoCompactionRetention" json:"autoCompactionRetention,omitempty"` - Metrics *string `yaml:"metrics" json:"metrics,omitempty"` - QuotaBackendBytes *int64 `yaml:"quotaBackendBytes" json:"quotaBackendBytes,omitempty"` - MaxRequestBytes *int64 `yaml:"maxRequestBytes" json:"maxRequestBytes,omitempty"` - MaxSnapshots *int `yaml:"maxSnapshots" json:"maxSnapshots,omitempty"` - MaxWals *int `yaml:"maxWals" json:"maxWals,omitempty"` - LogLevel *string `yaml:"logLevel" json:"logLevel"` -} - -// ExternalEtcd describes how to connect to an external etcd cluster -// KubeKey, Kubeadm and External are mutually exclusive -type ExternalEtcd struct { - // Endpoints of etcd members. Useful for using external etcd. - // If not provided, kubeadm will run etcd in a static pod. - Endpoints []string `yaml:"endpoints" json:"endpoints,omitempty"` - // CAFile is an SSL Certificate Authority file used to secure etcd communication. - CAFile string `yaml:"caFile" json:"caFile,omitempty"` - // CertFile is an SSL certification file used to secure etcd communication. - CertFile string `yaml:"certFile" json:"certFile,omitempty"` - // KeyFile is an SSL key file used to secure etcd communication. - KeyFile string `yaml:"keyFile" json:"keyFile,omitempty"` -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go b/cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go deleted file mode 100644 index c9f05581..00000000 --- a/cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go +++ /dev/null @@ -1,105 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1alpha2 - -import "k8s.io/apimachinery/pkg/runtime" - -// Kubernetes contains the configuration for the cluster -type Kubernetes struct { - Type string `yaml:"type" json:"type,omitempty"` - Version string `yaml:"version" json:"version,omitempty"` - ClusterName string `yaml:"clusterName" json:"clusterName,omitempty"` - DNSDomain string `yaml:"dnsDomain" json:"dnsDomain,omitempty"` - DisableKubeProxy bool `yaml:"disableKubeProxy" json:"disableKubeProxy,omitempty"` - MasqueradeAll bool `yaml:"masqueradeAll" json:"masqueradeAll,omitempty"` - MaxPods int `yaml:"maxPods" json:"maxPods,omitempty"` - PodPidsLimit int `yaml:"podPidsLimit" json:"podPidsLimit,omitempty"` - NodeCidrMaskSize int `yaml:"nodeCidrMaskSize" json:"nodeCidrMaskSize,omitempty"` - ApiserverCertExtraSans []string `yaml:"apiserverCertExtraSans" json:"apiserverCertExtraSans,omitempty"` - ProxyMode string `yaml:"proxyMode" json:"proxyMode,omitempty"` - AutoRenewCerts *bool `yaml:"autoRenewCerts" json:"autoRenewCerts,omitempty"` - // +optional - Nodelocaldns *bool `yaml:"nodelocaldns" json:"nodelocaldns,omitempty"` - ContainerManager string `yaml:"containerManager" json:"containerManager,omitempty"` - ContainerRuntimeEndpoint string `yaml:"containerRuntimeEndpoint" json:"containerRuntimeEndpoint,omitempty"` - NodeFeatureDiscovery NodeFeatureDiscovery `yaml:"nodeFeatureDiscovery" json:"nodeFeatureDiscovery,omitempty"` - Kata Kata `yaml:"kata" json:"kata,omitempty"` - ApiServerArgs []string `yaml:"apiserverArgs" json:"apiserverArgs,omitempty"` - ControllerManagerArgs []string `yaml:"controllerManagerArgs" json:"controllerManagerArgs,omitempty"` - SchedulerArgs []string `yaml:"schedulerArgs" json:"schedulerArgs,omitempty"` - KubeletArgs []string `yaml:"kubeletArgs" json:"kubeletArgs,omitempty"` - KubeProxyArgs []string `yaml:"kubeProxyArgs" json:"kubeProxyArgs,omitempty"` - FeatureGates map[string]bool `yaml:"featureGates" json:"featureGates,omitempty"` - KubeletConfiguration runtime.RawExtension `yaml:"kubeletConfiguration" json:"kubeletConfiguration,omitempty"` - KubeProxyConfiguration runtime.RawExtension `yaml:"kubeProxyConfiguration" json:"kubeProxyConfiguration,omitempty"` - Audit Audit `yaml:"audit" json:"audit,omitempty"` -} - -// Kata contains the configuration for the kata in cluster -type Kata struct { - Enabled *bool `yaml:"enabled" json:"enabled,omitempty"` -} - -// NodeFeatureDiscovery contains the configuration for the node-feature-discovery in cluster -type NodeFeatureDiscovery struct { - Enabled *bool `yaml:"enabled" json:"enabled,omitempty"` -} - -// Audit contains the configuration for the kube-apiserver audit in cluster -type Audit struct { - Enabled *bool `yaml:"enabled" json:"enabled,omitempty"` -} - -// EnableNodelocaldns is used to determine whether to deploy nodelocaldns. -func (k *Kubernetes) EnableNodelocaldns() bool { - if k.Nodelocaldns == nil { - return true - } - return *k.Nodelocaldns -} - -// EnableKataDeploy is used to determine whether to deploy kata. -func (k *Kubernetes) EnableKataDeploy() bool { - if k.Kata.Enabled == nil { - return false - } - return *k.Kata.Enabled -} - -// EnableNodeFeatureDiscovery is used to determine whether to deploy node-feature-discovery. -func (k *Kubernetes) EnableNodeFeatureDiscovery() bool { - if k.NodeFeatureDiscovery.Enabled == nil { - return false - } - return *k.NodeFeatureDiscovery.Enabled -} - -// EnableAutoRenewCerts is used to determine whether to enable AutoRenewCerts. -func (k *Kubernetes) EnableAutoRenewCerts() bool { - if k.AutoRenewCerts == nil { - return false - } - return *k.AutoRenewCerts -} - -// EnableAudit is used to determine whether to enable kube-apiserver audit. -func (k *Kubernetes) EnableAudit() bool { - if k.Audit.Enabled == nil { - return false - } - return *k.AutoRenewCerts -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/manifest_types.go b/cmd/kk/apis/kubekey/v1alpha2/manifest_types.go deleted file mode 100644 index 66aab584..00000000 --- a/cmd/kk/apis/kubekey/v1alpha2/manifest_types.go +++ /dev/null @@ -1,120 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha2 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -type Iso struct { - LocalPath string `yaml:"localPath" json:"localPath"` - Url string `yaml:"url" json:"url"` -} - -type Repository struct { - Iso Iso `yaml:"iso" json:"iso"` -} - -type OperatingSystem struct { - Arch string `yaml:"arch" json:"arch"` - Type string `yaml:"type" json:"type,omitempty"` - Id string `yaml:"id" json:"id"` - Version string `yaml:"version" json:"version"` - OsImage string `yaml:"osImage" json:"osImage"` - Repository Repository `yaml:"repository" json:"repository"` -} - -type KubernetesDistribution struct { - Type string `yaml:"type" json:"type"` - Version string `yaml:"version" json:"version"` -} - -type Helm struct { - Version string `yaml:"version" json:"version"` -} - -type CNI struct { - Version string `yaml:"version" json:"version"` -} - -type ETCD struct { - Version string `yaml:"version" json:"version"` -} - -type DockerManifest struct { - Version string `yaml:"version" json:"version"` -} - -type Crictl struct { - Version string `yaml:"version" json:"version"` -} - -type ContainerRuntime struct { - Type string `yaml:"type" json:"type"` - Version string `yaml:"version" json:"version"` -} - -type DockerRegistry struct { - Version string `yaml:"version" json:"version"` -} - -type Harbor struct { - Version string `yaml:"version" json:"version"` -} - -type DockerCompose struct { - Version string `yaml:"version" json:"version"` -} - -type Calicoctl struct { - Version string `yaml:"version" json:"version"` -} - -type Components struct { - Helm Helm `yaml:"helm" json:"helm"` - CNI CNI `yaml:"cni" json:"cni"` - ETCD ETCD `yaml:"etcd" json:"etcd"` - ContainerRuntimes []ContainerRuntime `yaml:"containerRuntimes" json:"containerRuntimes"` - Crictl Crictl `yaml:"crictl" json:"crictl,omitempty"` - DockerRegistry DockerRegistry `yaml:"docker-registry" json:"docker-registry"` - Harbor Harbor `yaml:"harbor" json:"harbor"` - DockerCompose DockerCompose `yaml:"docker-compose" json:"docker-compose"` - Calicoctl Calicoctl `yaml:"calicoctl" json:"calicoctl"` -} - -type ManifestRegistry struct { - Auths runtime.RawExtension `yaml:"auths" json:"auths,omitempty"` -} - -// ManifestSpec defines the desired state of Manifest -type ManifestSpec struct { - Arches []string `yaml:"arches" json:"arches"` - OperatingSystems []OperatingSystem `yaml:"operatingSystems" json:"operatingSystems"` - KubernetesDistributions []KubernetesDistribution `yaml:"kubernetesDistributions" json:"kubernetesDistributions"` - Components Components `yaml:"components" json:"components"` - Images []string `yaml:"images" json:"images"` - ManifestRegistry ManifestRegistry `yaml:"registry" json:"registry"` -} - -// Manifest is the Schema for the manifests API -type Manifest struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec ManifestSpec `json:"spec,omitempty"` -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/network_types.go b/cmd/kk/apis/kubekey/v1alpha2/network_types.go deleted file mode 100644 index af0b1b6f..00000000 --- a/cmd/kk/apis/kubekey/v1alpha2/network_types.go +++ /dev/null @@ -1,209 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1alpha2 - -type NetworkConfig struct { - Plugin string `yaml:"plugin" json:"plugin,omitempty"` - KubePodsCIDR string `yaml:"kubePodsCIDR" json:"kubePodsCIDR,omitempty"` - KubeServiceCIDR string `yaml:"kubeServiceCIDR" json:"kubeServiceCIDR,omitempty"` - Calico CalicoCfg `yaml:"calico" json:"calico,omitempty"` - Flannel FlannelCfg `yaml:"flannel" json:"flannel,omitempty"` - Kubeovn KubeovnCfg `yaml:"kubeovn" json:"kubeovn,omitempty"` - MultusCNI MultusCNI `yaml:"multusCNI" json:"multusCNI,omitempty"` - Hybridnet HybridnetCfg `yaml:"hybridnet" json:"hybridnet,omitempty"` -} - -type CalicoCfg struct { - IPIPMode string `yaml:"ipipMode" json:"ipipMode,omitempty"` - VXLANMode string `yaml:"vxlanMode" json:"vxlanMode,omitempty"` - VethMTU int `yaml:"vethMTU" json:"vethMTU,omitempty"` - Ipv4NatOutgoing *bool `yaml:"ipv4NatOutgoing" json:"ipv4NatOutgoing,omitempty"` - DefaultIPPOOL *bool `yaml:"defaultIPPOOL" json:"defaultIPPOOL,omitempty"` - EnableTypha *bool `yaml:"enableTypha" json:"enableTypha,omitempty"` -} - -type FlannelCfg struct { - BackendMode string `yaml:"backendMode" json:"backendMode,omitempty"` - Directrouting bool `yaml:"directRouting" json:"directRouting,omitempty"` -} - -type KubeovnCfg struct { - EnableSSL bool `yaml:"enableSSL" json:"enableSSL,omitempty"` - JoinCIDR string `yaml:"joinCIDR" json:"joinCIDR,omitempty"` - Label string `yaml:"label" json:"label,omitempty"` - TunnelType string `yaml:"tunnelType" json:"tunnelType,omitempty"` - SvcYamlIpfamilypolicy string `yaml:"svcYamlIpfamilypolicy" json:"svcYamlIpfamilypolicy,omitempty"` - Dpdk Dpdk `yaml:"dpdk" json:"dpdk,omitempty"` - OvsOvn OvsOvn `yaml:"ovs-ovn" json:"ovs-ovn,omitempty"` - KubeOvnController KubeOvnController `yaml:"kube-ovn-controller" json:"kube-ovn-controller,omitempty"` - KubeOvnCni KubeOvnCni `yaml:"kube-ovn-cni" json:"kube-ovn-cni,omitempty"` - KubeOvnPinger KubeOvnPinger `yaml:"kube-ovn-pinger" json:"kube-ovn-pinger,omitempty"` -} - -type Dpdk struct { - DpdkMode bool `yaml:"dpdkMode" json:"dpdkMode,omitempty"` - DpdkTunnelIface string `yaml:"dpdkTunnelIface" json:"dpdkTunnelIface,omitempty"` - DpdkVersion string `yaml:"dpdkVersion" json:"dpdkVersion,omitempty"` -} - -type OvsOvn struct { - HwOffload bool `yaml:"hwOffload" json:"hwOffload,omitempty"` -} - -type KubeOvnController struct { - PodGateway string `yaml:"podGateway" json:"podGateway,omitempty"` - CheckGateway *bool `yaml:"checkGateway" json:"checkGateway,omitempty"` - LogicalGateway bool `yaml:"logicalGateway" json:"logicalGateway,omitempty"` - ExcludeIps string `yaml:"excludeIps" json:"excludeIps,omitempty"` - NetworkType string `yaml:"networkType" json:"networkType,omitempty"` - VlanInterfaceName string `yaml:"vlanInterfaceName" json:"vlanInterfaceName,omitempty"` - VlanID string `yaml:"vlanID" json:"vlanID,omitempty"` - PodNicType string `yaml:"podNicType" json:"podNicType,omitempty"` - EnableLB *bool `yaml:"enableLB" json:"enableLB,omitempty"` - EnableNP *bool `yaml:"enableNP" json:"enableNP,omitempty"` - EnableEipSnat *bool `yaml:"enableEipSnat" json:"enableEipSnat,omitempty"` - EnableExternalVPC *bool `yaml:"enableExternalVPC" json:"enableExternalVPC,omitempty"` -} - -type KubeOvnCni struct { - EnableMirror bool `yaml:"enableMirror" json:"enableMirror,omitempty"` - Iface string `yaml:"iface" json:"iface,omitempty"` - CNIConfigPriority string `yaml:"CNIConfigPriority" json:"CNIConfigPriority,omitempty"` - Modules string `yaml:"modules" json:"modules,omitempty"` - RPMs string `yaml:"RPMs" json:"RPMs,omitempty"` -} - -type KubeOvnPinger struct { - PingerExternalAddress string `yaml:"pingerExternalAddress" json:"pingerExternalAddress,omitempty"` - PingerExternalDomain string `yaml:"pingerExternalDomain" json:"pingerExternalDomain,omitempty"` -} - -type HybridnetCfg struct { - DefaultNetworkType string `yaml:"defaultNetworkType" json:"defaultNetworkType,omitempty"` - EnableNetworkPolicy *bool `yaml:"enableNetworkPolicy" json:"enableNetworkPolicy,omitempty"` - Init *bool `yaml:"init" json:"init,omitempty"` - PreferVxlanInterfaces string `yaml:"preferVxlanInterfaces" json:"preferVxlanInterfaces,omitempty"` - PreferVlanInterfaces string `yaml:"preferVlanInterfaces" json:"preferVlanInterfaces,omitempty"` - PreferBGPInterfaces string `yaml:"preferBGPInterfaces" json:"preferBGPInterfaces,omitempty"` - Networks []HybridnetNetwork `yaml:"networks" json:"networks,omitempty"` -} - -type HybridnetNetwork struct { - Name string `yaml:"name" json:"name,omitempty"` - NetID *int `yaml:"netID" json:"netID,omitempty"` - Type string `yaml:"type" json:"type,omitempty"` - Mode string `yaml:"mode" json:"mode,omitempty"` - NodeSelector map[string]string `yaml:"nodeSelector" json:"nodeSelector,omitempty"` - Subnets []HybridnetSubnet `yaml:"subnets" json:"subnets,omitempty"` -} - -type HybridnetSubnet struct { - Name string `yaml:"name" json:"name,omitempty"` - NetID *int `yaml:"netID" json:"netID,omitempty"` - CIDR string `yaml:"cidr" json:"cidr,omitempty"` - Gateway string `yaml:"gateway" json:"gateway,omitempty"` - Start string `yaml:"start" json:"start,omitempty"` - End string `yaml:"end" json:"end,omitempty"` - ReservedIPs []string `yaml:"reservedIPs" json:"reservedIPs,omitempty"` - ExcludeIPs []string `yaml:"excludeIPs" json:"excludeIPs,omitempty"` -} - -func (k *KubeovnCfg) KubeovnCheckGateway() bool { - if k.KubeOvnController.CheckGateway == nil { - return true - } - return *k.KubeOvnController.CheckGateway -} - -func (k *KubeovnCfg) KubeovnEnableLB() bool { - if k.KubeOvnController.EnableLB == nil { - return true - } - return *k.KubeOvnController.EnableLB -} - -func (k *KubeovnCfg) KubeovnEnableNP() bool { - if k.KubeOvnController.EnableNP == nil { - return true - } - return *k.KubeOvnController.EnableNP -} - -func (k *KubeovnCfg) KubeovnEnableEipSnat() bool { - if k.KubeOvnController.EnableEipSnat == nil { - return true - } - return *k.KubeOvnController.EnableEipSnat -} - -func (k *KubeovnCfg) KubeovnEnableExternalVPC() bool { - if k.KubeOvnController.EnableExternalVPC == nil { - return true - } - return *k.KubeOvnController.EnableExternalVPC -} - -type MultusCNI struct { - Enabled *bool `yaml:"enabled" json:"enabled,omitempty"` -} - -func (n *NetworkConfig) EnableMultusCNI() bool { - if n.MultusCNI.Enabled == nil { - return false - } - return *n.MultusCNI.Enabled -} - -// EnableIPV4POOL_NAT_OUTGOING is used to determine whether to enable CALICO_IPV4POOL_NAT_OUTGOING. -func (c *CalicoCfg) EnableIPV4POOL_NAT_OUTGOING() bool { - if c.Ipv4NatOutgoing == nil { - return true - } - return *c.Ipv4NatOutgoing -} - -// EnableDefaultIPPOOL is used to determine whether to create default ippool -func (c *CalicoCfg) EnableDefaultIPPOOL() bool { - if c.DefaultIPPOOL == nil { - return true - } - return *c.DefaultIPPOOL -} - -// Typha is used to determine whether to enable calico Typha -func (c *CalicoCfg) Typha() bool { - if c.EnableTypha == nil { - return false - } - return *c.EnableTypha -} - -// EnableInit is used to determine whether to create default network -func (h *HybridnetCfg) EnableInit() bool { - if h.Init == nil { - return true - } - return *h.Init -} - -// NetworkPolicy is used to determine whether to enable network policy -func (h *HybridnetCfg) NetworkPolicy() bool { - if h.EnableNetworkPolicy == nil { - return true - } - return *h.EnableNetworkPolicy -} diff --git a/cmd/kk/apis/kubekey/v1alpha2/storage_types.go b/cmd/kk/apis/kubekey/v1alpha2/storage_types.go deleted file mode 100644 index a30f51dc..00000000 --- a/cmd/kk/apis/kubekey/v1alpha2/storage_types.go +++ /dev/null @@ -1,25 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1alpha2 - -type StorageConfig struct { - OpenEBS OpenEBSCfg `yaml:"openebs" json:"openebs,omitempty"` -} - -type OpenEBSCfg struct { - BasePath string `yaml:"basePath" json:"basePath,omitempty"` -} diff --git a/cmd/kk/cmd/add/add.go b/cmd/kk/cmd/add/add.go deleted file mode 100644 index f4d0f4d4..00000000 --- a/cmd/kk/cmd/add/add.go +++ /dev/null @@ -1,47 +0,0 @@ -/* - Copyright 2020 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package add - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type AddOptions struct { - CommonOptions *options.CommonOptions -} - -func NewAddOptions() *AddOptions { - return &AddOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdAdd creates a new add command -func NewCmdAdd() *cobra.Command { - o := NewAddOptions() - cmd := &cobra.Command{ - Use: "add", - Short: "Add nodes to kubernetes cluster", - } - - o.CommonOptions.AddCommonFlag(cmd) - - cmd.AddCommand(NewCmdAddNodes()) - return cmd -} diff --git a/cmd/kk/cmd/add/add_nodes.go b/cmd/kk/cmd/add/add_nodes.go deleted file mode 100644 index 1caee472..00000000 --- a/cmd/kk/cmd/add/add_nodes.go +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package add - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type AddNodesOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - SkipPullImages bool - ContainerManager string - DownloadCmd string - Artifact string - InstallPackages bool -} - -func NewAddNodesOptions() *AddNodesOptions { - return &AddNodesOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdAddNodes creates a new add nodes command -func NewCmdAddNodes() *cobra.Command { - o := NewAddNodesOptions() - cmd := &cobra.Command{ - Use: "nodes", - Short: "Add nodes to the cluster according to the new nodes information from the specified configuration file", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *AddNodesOptions) Complete(_ *cobra.Command, _ []string) error { - if o.Artifact == "" { - o.InstallPackages = false - } - return nil -} - -func (o *AddNodesOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KsEnable: false, - Debug: o.CommonOptions.Verbose, - IgnoreErr: o.CommonOptions.IgnoreErr, - SkipConfirmCheck: o.CommonOptions.SkipConfirmCheck, - SkipPullImages: o.SkipPullImages, - ContainerManager: o.ContainerManager, - Artifact: o.Artifact, - InstallPackages: o.InstallPackages, - Namespace: o.CommonOptions.Namespace, - } - return pipelines.AddNodes(arg, o.DownloadCmd) -} - -func (o *AddNodesOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().BoolVarP(&o.SkipPullImages, "skip-pull-images", "", false, "Skip pre pull images") - cmd.Flags().StringVarP(&o.ContainerManager, "container-manager", "", "docker", "Container manager: docker, crio, containerd and isula.") - cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", - `The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`) - cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact") - cmd.Flags().BoolVarP(&o.InstallPackages, "with-packages", "", false, "install operation system packages by artifact") -} diff --git a/cmd/kk/cmd/alpha/alpha.go b/cmd/kk/cmd/alpha/alpha.go deleted file mode 100644 index 44fb0bea..00000000 --- a/cmd/kk/cmd/alpha/alpha.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package alpha - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/alpha/cri" -) - -// NewAlphaCmd create a new Alpha command -func NewAlphaCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "alpha", - Short: "Commands for features in alpha", - } - - cmd.AddCommand(NewCmdCreate()) - cmd.AddCommand(NewCmdUpgrade()) - cmd.AddCommand(cri.NewCmdCri()) - return cmd -} diff --git a/cmd/kk/cmd/alpha/create.go b/cmd/kk/cmd/alpha/create.go deleted file mode 100644 index c3ee724d..00000000 --- a/cmd/kk/cmd/alpha/create.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package alpha - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/create/phase" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type CreateOptions struct { - CommonOptions *options.CommonOptions -} - -func NewCreateOptions() *CreateOptions { - return &CreateOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreate creates a new create command -func NewCmdCreate() *cobra.Command { - o := NewCreateOptions() - cmd := &cobra.Command{ - Use: "create", - Short: "Create a cluster by phase cmds for testing", - } - - o.CommonOptions.AddCommonFlag(cmd) - cmd.AddCommand(phase.NewPhaseCommand()) - return cmd -} diff --git a/cmd/kk/cmd/alpha/cri/cri.go b/cmd/kk/cmd/alpha/cri/cri.go deleted file mode 100644 index a9b51c4d..00000000 --- a/cmd/kk/cmd/alpha/cri/cri.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cri - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type MigrateOptions struct { - CommonOptions *options.CommonOptions -} - -func NewMigrateOptions() *MigrateOptions { - return &MigrateOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCri creates a new Migrate command -func NewCmdCri() *cobra.Command { - o := NewMigrateOptions() - cmd := &cobra.Command{ - Use: "cri", - Short: "cri", - } - - o.CommonOptions.AddCommonFlag(cmd) - cmd.AddCommand(NewCmdMigrateCri()) - return cmd -} diff --git a/cmd/kk/cmd/alpha/cri/migrate_cri.go b/cmd/kk/cmd/alpha/cri/migrate_cri.go deleted file mode 100644 index c11ec145..00000000 --- a/cmd/kk/cmd/alpha/cri/migrate_cri.go +++ /dev/null @@ -1,104 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cri - -import ( - "github.com/pkg/errors" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type MigrateCriOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string - EnableKubeSphere bool - KubeSphere string - DownloadCmd string - Artifact string - Type string - Role string -} - -func NewMigrateCriOptions() *MigrateCriOptions { - return &MigrateCriOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdMigrateCri creates a new delete cluster command -func NewCmdMigrateCri() *cobra.Command { - o := NewMigrateCriOptions() - cmd := &cobra.Command{ - Use: "migrate", - Short: "Migrate a container", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Validate()) - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *MigrateCriOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - KubernetesVersion: o.Kubernetes, - Type: o.Type, - Role: o.Role, - } - return pipelines.MigrateCri(arg, o.DownloadCmd) -} - -func (o *MigrateCriOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.Role, "role", "", "", "Role groups for migrating. Support: master, worker, all.") - cmd.Flags().StringVarP(&o.Type, "type", "", "", "Type of target CRI. Support: docker, containerd.") - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", - `The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`) - cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact") -} - -func (o *MigrateCriOptions) Validate() error { - if o.Role == "" { - return errors.New("node Role can not be empty") - } - if o.Role != common.Worker && o.Role != common.Master && o.Role != "all" { - - return errors.Errorf("node Role is invalid: %s", o.Role) - } - if o.Type == "" { - return errors.New("cri Type can not be empty") - } - if o.Type != common.Docker && o.Type != common.Containerd { - return errors.Errorf("cri Type is invalid: %s", o.Type) - } - if o.ClusterCfgFile == "" { - return errors.New("configuration file can not be empty") - } - return nil -} diff --git a/cmd/kk/cmd/alpha/upgrade.go b/cmd/kk/cmd/alpha/upgrade.go deleted file mode 100644 index 09d739ca..00000000 --- a/cmd/kk/cmd/alpha/upgrade.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package alpha - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/upgrade/phase" -) - -type UpgradeOptions struct { - CommonOptions *options.CommonOptions -} - -func NewUpgradeOptions() *UpgradeOptions { - return &UpgradeOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdUpgrade creates a new upgrade command -func NewCmdUpgrade() *cobra.Command { - o := NewUpgradeOptions() - cmd := &cobra.Command{ - Use: "upgrade", - Short: "Upgrade your cluster by phase cmd for testing", - } - o.CommonOptions.AddCommonFlag(cmd) - cmd.AddCommand(phase.NewPhaseCommand()) - return cmd -} diff --git a/cmd/kk/cmd/artifact/artifact.go b/cmd/kk/cmd/artifact/artifact.go deleted file mode 100755 index 8f046c98..00000000 --- a/cmd/kk/cmd/artifact/artifact.go +++ /dev/null @@ -1,50 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package artifact - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/artifact/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type ArtifactOptions struct { - CommonOptions *options.CommonOptions -} - -func NewArtifactOptions() *ArtifactOptions { - return &ArtifactOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdArtifact creates a new cobra.Command for `kubekey artifact` -func NewCmdArtifact() *cobra.Command { - o := NewArtifactOptions() - cmd := &cobra.Command{ - Use: "artifact", - Short: "Manage a KubeKey offline installation package", - } - - o.CommonOptions.AddCommonFlag(cmd) - - cmd.AddCommand(NewCmdArtifactExport()) - cmd.AddCommand(images.NewCmdArtifactImages()) - cmd.AddCommand(NewCmdArtifactImport()) - return cmd -} diff --git a/cmd/kk/cmd/artifact/export.go b/cmd/kk/cmd/artifact/export.go deleted file mode 100644 index b5c2b822..00000000 --- a/cmd/kk/cmd/artifact/export.go +++ /dev/null @@ -1,94 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package artifact - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type ArtifactExportOptions struct { - CommonOptions *options.CommonOptions - - ManifestFile string - Output string - CriSocket string - DownloadCmd string -} - -func NewArtifactExportOptions() *ArtifactExportOptions { - return &ArtifactExportOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdArtifactExport creates a new `kubekey artifact export` command -func NewCmdArtifactExport() *cobra.Command { - o := NewArtifactExportOptions() - cmd := &cobra.Command{ - Use: "export", - Short: "Export a KubeKey offline installation package", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Validate(args)) - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *ArtifactExportOptions) Complete(_ *cobra.Command, _ []string) error { - if o.Output == "" { - o.Output = "kubekey-artifact.tar.gz" - } - return nil -} - -func (o *ArtifactExportOptions) Validate(_ []string) error { - if o.ManifestFile == "" { - return fmt.Errorf("--manifest can not be an empty string") - } - return nil -} - -func (o *ArtifactExportOptions) Run() error { - arg := common.ArtifactArgument{ - ManifestFile: o.ManifestFile, - Output: o.Output, - CriSocket: o.CriSocket, - Debug: o.CommonOptions.Verbose, - IgnoreErr: o.CommonOptions.IgnoreErr, - } - - return pipelines.ArtifactExport(arg, o.DownloadCmd) -} - -func (o *ArtifactExportOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ManifestFile, "manifest", "m", "", "Path to a manifest file") - cmd.Flags().StringVarP(&o.Output, "output", "o", "", "Path to a output path") - cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", - `The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`) -} diff --git a/cmd/kk/cmd/artifact/images/images.go b/cmd/kk/cmd/artifact/images/images.go deleted file mode 100644 index 0d3ae5eb..00000000 --- a/cmd/kk/cmd/artifact/images/images.go +++ /dev/null @@ -1,48 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type ArtifactImagesOptions struct { - CommonOptions *options.CommonOptions -} - -func NewArtifactImagesOptions() *ArtifactImagesOptions { - return &ArtifactImagesOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdArtifactImages creates a new `kubekey artifact image` command -func NewCmdArtifactImages() *cobra.Command { - o := NewArtifactImagesOptions() - cmd := &cobra.Command{ - Use: "images", - Aliases: []string{"image", "i"}, - Short: "manage KubeKey artifact image", - } - - o.CommonOptions.AddCommonFlag(cmd) - cmd.AddCommand(NewCmdArtifactImagesPush()) - - return cmd -} diff --git a/cmd/kk/cmd/artifact/images/push.go b/cmd/kk/cmd/artifact/images/push.go deleted file mode 100644 index d337bdd1..00000000 --- a/cmd/kk/cmd/artifact/images/push.go +++ /dev/null @@ -1,140 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "os" - "path/filepath" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" -) - -type ArtifactImagesPushOptions struct { - CommonOptions *options.CommonOptions - - ImageDirPath string - Artifact string - ClusterCfgFile string -} - -func NewArtifactImagesPushOptions() *ArtifactImagesPushOptions { - return &ArtifactImagesPushOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdArtifactImagesPush creates a new `kubekey artifacts images push` command -func NewCmdArtifactImagesPush() *cobra.Command { - o := NewArtifactImagesPushOptions() - cmd := &cobra.Command{ - Use: "push", - Short: "push images to a registry from an artifact", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Validate(args)) - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *ArtifactImagesPushOptions) Complete(_ *cobra.Command, _ []string) error { - if o.ImageDirPath == "" && o.Artifact == "" { - currentDir, err := filepath.Abs(filepath.Dir(os.Args[0])) - if err != nil { - return errors.Wrap(err, "failed to get current directory") - } - o.ImageDirPath = filepath.Join(currentDir, "kubekey", "images") - } - - return nil -} - -func (o *ArtifactImagesPushOptions) Validate(_ []string) error { - if o.ClusterCfgFile == "" { - return errors.New("kubekey config file is required") - } - if o.ImageDirPath != "" && o.Artifact != "" { - return errors.New("only one of --image-dir or --artifact can be specified") - } - return nil -} - -func (o *ArtifactImagesPushOptions) Run() error { - arg := common.Argument{ - ImagesDir: o.ImageDirPath, - Artifact: o.Artifact, - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - IgnoreErr: o.CommonOptions.IgnoreErr, - } - return runPush(arg) -} - -func (o *ArtifactImagesPushOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ImageDirPath, "images-dir", "", "", "Path to a KubeKey artifact images directory") - cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact") - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") -} - -func runPush(arg common.Argument) error { - runtime, err := common.NewKubeRuntime(common.File, arg) - if err != nil { - return err - } - - if err := newImagesPushPipeline(runtime); err != nil { - return err - } - - return nil -} - -func newImagesPushPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - - m := []module.Module{ - &artifact.UnArchiveModule{Skip: noArtifact}, - &images.CopyImagesToRegistryModule{ImagePath: runtime.Arg.ImagesDir}, - &filesystem.ChownWorkDirModule{}, - } - - p := pipeline.Pipeline{ - Name: "ArtifactImagesPushPipeline", - Modules: m, - Runtime: runtime, - } - - if err := p.Start(); err != nil { - return err - } - - return nil -} diff --git a/cmd/kk/cmd/artifact/import.go b/cmd/kk/cmd/artifact/import.go deleted file mode 100755 index 8707a99d..00000000 --- a/cmd/kk/cmd/artifact/import.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package artifact - -import ( - "errors" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/artifact" -) - -type ArtifactImportOptions struct { - CommonOptions *options.CommonOptions - Artifact string -} - -func NewArtifactImportOptions() *ArtifactImportOptions { - return &ArtifactImportOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdArtifactImport creates a new artifact import command -func NewCmdArtifactImport() *cobra.Command { - o := NewArtifactImportOptions() - cmd := &cobra.Command{ - Use: "import", - Short: "Import a KubeKey offline installation package", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Validate(args)) - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *ArtifactImportOptions) Run() error { - arg := common.Argument{ - Debug: o.CommonOptions.Verbose, - Artifact: o.Artifact, - } - return artifact.ArtifactImport(arg) -} - -func (o *ArtifactImportOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a artifact gzip") -} - -func (o *ArtifactImportOptions) Validate(_ []string) error { - if o.Artifact == "" { - return errors.New("artifact path can not be empty") - } - return nil -} diff --git a/cmd/kk/cmd/cert/cert.go b/cmd/kk/cmd/cert/cert.go deleted file mode 100644 index 89c21785..00000000 --- a/cmd/kk/cmd/cert/cert.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cert - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type CertOptions struct { - CommonOptions *options.CommonOptions -} - -func NewCertsOptions() *CertOptions { - return &CertOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCerts creates a new cert command -func NewCmdCerts() *cobra.Command { - o := NewCertsOptions() - cmd := &cobra.Command{ - Use: "certs", - Short: "cluster certs", - } - - o.CommonOptions.AddCommonFlag(cmd) - - cmd.AddCommand(NewCmdCertList()) - cmd.AddCommand(NewCmdCertRenew()) - return cmd -} diff --git a/cmd/kk/cmd/cert/list_cert.go b/cmd/kk/cmd/cert/list_cert.go deleted file mode 100644 index 79b58fd2..00000000 --- a/cmd/kk/cmd/cert/list_cert.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cert - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type CertListOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string -} - -func NewCertListOptions() *CertListOptions { - return &CertListOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCertList creates a new cert list command -func NewCmdCertList() *cobra.Command { - o := NewCertListOptions() - cmd := &cobra.Command{ - Use: "check-expiration", - Short: "Check certificates expiration for a Kubernetes cluster", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *CertListOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - } - return pipelines.CheckCerts(arg) -} - -func (o *CertListOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") -} diff --git a/cmd/kk/cmd/cert/renew_cert.go b/cmd/kk/cmd/cert/renew_cert.go deleted file mode 100644 index f3dcbd82..00000000 --- a/cmd/kk/cmd/cert/renew_cert.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cert - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type CertRenewOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string -} - -func NewCertRenewOptions() *CertRenewOptions { - return &CertRenewOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCertRenew creates a new cert renew command -func NewCmdCertRenew() *cobra.Command { - o := NewCertRenewOptions() - cmd := &cobra.Command{ - Use: "renew", - Short: "renew a cluster certs", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *CertRenewOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - } - return pipelines.RenewCerts(arg) -} - -func (o *CertRenewOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") -} diff --git a/cmd/kk/cmd/completion/completion.go b/cmd/kk/cmd/completion/completion.go deleted file mode 100644 index ff832f41..00000000 --- a/cmd/kk/cmd/completion/completion.go +++ /dev/null @@ -1,113 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package completion - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" -) - -// CompletionOptions is the option of completion command -type CompletionOptions struct { - Type string -} - -func NewCompletionOptions() *CompletionOptions { - return &CompletionOptions{} -} - -// ShellTypes contains all types of shell -var ShellTypes = []string{ - "zsh", "bash", "powerShell", -} - -var completionOptions CompletionOptions - -func NewCmdCompletion() *cobra.Command { - o := NewCompletionOptions() - cmd := &cobra.Command{ - Use: "completion", - Short: "Generate shell completion scripts", - Long: `Generate shell completion scripts -Normally you don't need to do more extra work to have this feature if you've installed kk by brew`, - Example: `# Installing bash completion on Linux -## If bash-completion is not installed on Linux, please install the 'bash-completion' package -## via your distribution's package manager. -## Load the ks completion code for bash into the current shell -source <(ks completion bash) -## Write bash completion code to a file and source if from .bash_profile -mkdir -p ~/.config/kk/ && kk completion --type bash > ~/.config/kk/completion.bash.inc -printf " -# kk shell completion -source '$HOME/.config/kk/completion.bash.inc' -" >> $HOME/.bash_profile -source $HOME/.bash_profile - -In order to have good experience on zsh completion, ohmyzsh is a good choice. -Please install ohmyzsh by the following command -sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" -Get more details about onmyzsh from https://github.com/ohmyzsh/ohmyzsh - -Load the kk completion code for zsh[1] into the current shell -source <(kk completion --type zsh) -Set the kk completion code for zsh[1] to autoload on startup -kk completion --type zsh > "${fpath[1]}/_kk"`, - Run: func(cmd *cobra.Command, _ []string) { - util.CheckErr(o.Run(cmd)) - }, - } - - o.AddFlags(cmd) - if err := completionSetting(cmd); err != nil { - panic(fmt.Sprintf("register flag type for sub-command doc failed %#v\n", err)) - } - return cmd -} - -func (o *CompletionOptions) Run(cmd *cobra.Command) error { - var err error - shellType := completionOptions.Type - switch shellType { - case "zsh": - err = cmd.GenZshCompletion(cmd.OutOrStdout()) - case "powerShell": - err = cmd.GenPowerShellCompletion(cmd.OutOrStdout()) - case "bash": - err = cmd.GenBashCompletion(cmd.OutOrStdout()) - case "": - err = cmd.Help() - default: - err = fmt.Errorf("unknown shell type %s", shellType) - } - return err -} - -func (o *CompletionOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&completionOptions.Type, "type", "t", "", - fmt.Sprintf("Generate different types of shell which are %v", ShellTypes)) -} - -func completionSetting(cmd *cobra.Command) error { - err := cmd.RegisterFlagCompletionFunc("type", func(cmd *cobra.Command, args []string, toComplete string) ( - i []string, directive cobra.ShellCompDirective) { - return ShellTypes, cobra.ShellCompDirectiveDefault - }) - return err -} diff --git a/cmd/kk/cmd/create/cluster.go b/cmd/kk/cmd/create/cluster.go deleted file mode 100644 index 3ca16ec0..00000000 --- a/cmd/kk/cmd/create/cluster.go +++ /dev/null @@ -1,163 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package create - -import ( - "fmt" - "time" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -type CreateClusterOptions struct { - CommonOptions *options.CommonOptions - - ClusterCfgFile string - Kubernetes string - EnableKubeSphere bool - KubeSphere string - LocalStorage bool - SkipPullImages bool - SkipPushImages bool - SecurityEnhancement bool - ContainerManager string - DownloadCmd string - Artifact string - InstallPackages bool - - localStorageChanged bool -} - -func NewCreateClusterOptions() *CreateClusterOptions { - return &CreateClusterOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreateCluster creates a new create cluster command -func NewCmdCreateCluster() *cobra.Command { - o := NewCreateClusterOptions() - cmd := &cobra.Command{ - Use: "cluster", - Short: "Create a Kubernetes or KubeSphere cluster", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Validate(cmd, args)) - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := completionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *CreateClusterOptions) Complete(cmd *cobra.Command, args []string) error { - var ksVersion string - if o.EnableKubeSphere && len(args) > 0 { - ksVersion = args[0] - } else { - ksVersion = kubesphere.Latest().Version - } - o.KubeSphere = ksVersion - - if o.Artifact == "" { - o.InstallPackages = false - o.SkipPushImages = false - } - - if cmd.Flags().Changed("with-local-storage") { - o.localStorageChanged = true - } - return nil -} - -func (o *CreateClusterOptions) Validate(_ *cobra.Command, _ []string) error { - switch o.ContainerManager { - case common.Docker, common.Containerd, common.Crio, common.Isula: - default: - return fmt.Errorf("unsupport container runtime [%s]", o.ContainerManager) - } - return nil -} - -func (o *CreateClusterOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - KsEnable: o.EnableKubeSphere, - KsVersion: o.KubeSphere, - SkipPullImages: o.SkipPullImages, - SkipPushImages: o.SkipPushImages, - SecurityEnhancement: o.SecurityEnhancement, - Debug: o.CommonOptions.Verbose, - IgnoreErr: o.CommonOptions.IgnoreErr, - SkipConfirmCheck: o.CommonOptions.SkipConfirmCheck, - ContainerManager: o.ContainerManager, - Artifact: o.Artifact, - InstallPackages: o.InstallPackages, - Namespace: o.CommonOptions.Namespace, - } - - if o.localStorageChanged { - deploy := o.LocalStorage - arg.DeployLocalStorage = &deploy - } - - return pipelines.CreateCluster(arg, o.DownloadCmd) -} - -func (o *CreateClusterOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().BoolVarP(&o.LocalStorage, "with-local-storage", "", false, "Deploy a local PV provisioner") - cmd.Flags().BoolVarP(&o.EnableKubeSphere, "with-kubesphere", "", false, fmt.Sprintf("Deploy a specific version of kubesphere (default %s)", kubesphere.Latest().Version)) - cmd.Flags().BoolVarP(&o.SkipPullImages, "skip-pull-images", "", false, "Skip pre pull images") - cmd.Flags().BoolVarP(&o.SkipPushImages, "skip-push-images", "", false, "Skip pre push images") - cmd.Flags().BoolVarP(&o.SecurityEnhancement, "with-security-enhancement", "", false, "Security enhancement") - cmd.Flags().StringVarP(&o.ContainerManager, "container-manager", "", "docker", "Container runtime: docker, crio, containerd and isula.") - cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", - `The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`) - cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact") - cmd.Flags().BoolVarP(&o.InstallPackages, "with-packages", "", false, "install operation system packages by artifact") -} - -func completionSetting(cmd *cobra.Command) (err error) { - cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ( - strings []string, directive cobra.ShellCompDirective) { - versionArray := kubesphere.VersionsStringArr() - versionArray = append(versionArray, time.Now().Add(-time.Hour*24).Format("nightly-20060102")) - return versionArray, cobra.ShellCompDirectiveNoFileComp - } - - err = cmd.RegisterFlagCompletionFunc("with-kubernetes", func(cmd *cobra.Command, args []string, toComplete string) ( - strings []string, directive cobra.ShellCompDirective) { - return kubernetes.SupportedK8sVersionList(), cobra.ShellCompDirectiveNoFileComp - }) - return -} diff --git a/cmd/kk/cmd/create/config.go b/cmd/kk/cmd/create/config.go deleted file mode 100644 index ab259b24..00000000 --- a/cmd/kk/cmd/create/config.go +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package create - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/config" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -type CreateConfigOptions struct { - CommonOptions *options.CommonOptions - Name string - ClusterCfgFile string - Kubernetes string - EnableKubeSphere bool - KubeSphere string - FromCluster bool - KubeConfig string -} - -func NewCreateConfigOptions() *CreateConfigOptions { - return &CreateConfigOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreateConfig creates a create config command -func NewCmdCreateConfig() *cobra.Command { - o := NewCreateConfigOptions() - cmd := &cobra.Command{ - Use: "config", - Short: "Create cluster configuration file", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Run()) - - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *CreateConfigOptions) Complete(cmd *cobra.Command, args []string) error { - var ksVersion string - if o.EnableKubeSphere && len(args) > 0 { - ksVersion = args[0] - } else { - ksVersion = kubesphere.Latest().Version - } - o.KubeSphere = ksVersion - return nil -} - -func (o *CreateConfigOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - KsEnable: o.EnableKubeSphere, - KsVersion: o.KubeSphere, - FromCluster: o.FromCluster, - KubeConfig: o.KubeConfig, - } - - return config.GenerateKubeKeyConfig(arg, o.Name) -} - -func (o *CreateConfigOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.Name, "name", "", "sample", "Specify a name of cluster object") - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Specify a configuration file path") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().BoolVarP(&o.EnableKubeSphere, "with-kubesphere", "", false, fmt.Sprintf("Deploy a specific version of kubesphere (default %s)", kubesphere.Latest().Version)) - cmd.Flags().BoolVarP(&o.FromCluster, "from-cluster", "", false, "Create a configuration based on existing cluster") - cmd.Flags().StringVarP(&o.KubeConfig, "kubeconfig", "", "", "Specify a kubeconfig file") -} diff --git a/cmd/kk/cmd/create/create.go b/cmd/kk/cmd/create/create.go deleted file mode 100644 index dfb826e8..00000000 --- a/cmd/kk/cmd/create/create.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package create - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type CreateOptions struct { - CommonOptions *options.CommonOptions -} - -func NewCreateOptions() *CreateOptions { - return &CreateOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreate creates a new create command -func NewCmdCreate() *cobra.Command { - o := NewCreateOptions() - cmd := &cobra.Command{ - Use: "create", - Short: "Create a cluster or a cluster configuration file", - } - - o.CommonOptions.AddCommonFlag(cmd) - - cmd.AddCommand(NewCmdCreateCluster()) - cmd.AddCommand(NewCmdCreateConfig()) - cmd.AddCommand(NewCmdCreateManifest()) - return cmd -} diff --git a/cmd/kk/cmd/create/manifest.go b/cmd/kk/cmd/create/manifest.go deleted file mode 100644 index 4ede87a3..00000000 --- a/cmd/kk/cmd/create/manifest.go +++ /dev/null @@ -1,96 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package create - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - "k8s.io/client-go/util/homedir" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" -) - -type CreateManifestOptions struct { - CommonOptions *options.CommonOptions - - Name string - KubeConfig string - FileName string -} - -func NewCreateManifestOptions() *CreateManifestOptions { - return &CreateManifestOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreateManifest creates a create manifest command -func NewCmdCreateManifest() *cobra.Command { - o := NewCreateManifestOptions() - cmd := &cobra.Command{ - Use: "manifest", - Short: "Create an offline installation package configuration file", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Run()) - - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *CreateManifestOptions) Complete(cmd *cobra.Command, args []string) error { - if o.Name != "" { - o.Name = strings.Split(o.Name, ".")[0] - } - if o.KubeConfig == "" { - o.KubeConfig = filepath.Join(homedir.HomeDir(), ".kube", "config") - } - if o.FileName == "" { - currentDir, err := filepath.Abs(filepath.Dir(os.Args[0])) - if err != nil { - return errors.Wrap(err, "Failed to get current dir") - } - o.FileName = filepath.Join(currentDir, fmt.Sprintf("manifest-%s.yaml", o.Name)) - } - return nil -} - -func (o *CreateManifestOptions) Run() error { - arg := common.Argument{ - FilePath: o.FileName, - KubeConfig: o.KubeConfig, - } - return artifact.CreateManifest(arg, o.Name) -} - -func (o *CreateManifestOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.Name, "name", "", "sample", "Specify a name of manifest object") - cmd.Flags().StringVarP(&o.FileName, "filename", "f", "", "Specify a manifest file path") - cmd.Flags().StringVar(&o.KubeConfig, "kubeconfig", "", "Specify a kubeconfig file") -} diff --git a/cmd/kk/cmd/create/phase/binary.go b/cmd/kk/cmd/create/phase/binary.go deleted file mode 100755 index f9d4b4ab..00000000 --- a/cmd/kk/cmd/create/phase/binary.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/binary" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes" -) - -type CreateBinaryOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string - DownloadCmd string -} - -func NewCreateBinaryOptions() *CreateBinaryOptions { - return &CreateBinaryOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreateBinary creates a new artifact import command -func NewCmdCreateBinary() *cobra.Command { - o := NewCreateBinaryOptions() - cmd := &cobra.Command{ - Use: "binary", - Short: "Download the binaries on the local", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - if err := k8sCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *CreateBinaryOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - Debug: o.CommonOptions.Verbose, - } - return binary.CreateBinary(arg, o.DownloadCmd) -} - -func (o *CreateBinaryOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", - `The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`) - -} - -func k8sCompletionSetting(cmd *cobra.Command) (err error) { - err = cmd.RegisterFlagCompletionFunc("with-kubernetes", func(cmd *cobra.Command, args []string, toComplete string) ( - strings []string, directive cobra.ShellCompDirective) { - return kubernetes.SupportedK8sVersionList(), cobra.ShellCompDirectiveNoFileComp - }) - return -} diff --git a/cmd/kk/cmd/create/phase/configure.go b/cmd/kk/cmd/create/phase/configure.go deleted file mode 100644 index 635f2ea4..00000000 --- a/cmd/kk/cmd/create/phase/configure.go +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/kubernetes" -) - -type CreateConfigureKubernetesOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string - LocalStorage bool - - localStorageChanged bool -} - -func NewCreateConfigureKubernetesOptions() *CreateConfigureKubernetesOptions { - return &CreateConfigureKubernetesOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -func (o *CreateConfigureKubernetesOptions) Complete(cmd *cobra.Command, args []string) error { - if cmd.Flags().Changed("with-local-storage") { - o.localStorageChanged = true - } - return nil -} - -// NewCmdCreateConfigureKubernetes creates a new CreateConfigureKubernetes command -func NewCmdCreateConfigureKubernetes() *cobra.Command { - o := NewCreateConfigureKubernetesOptions() - cmd := &cobra.Command{ - Use: "configure", - Short: "Configure the k8s cluster with plugins, certs and PV", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := k8sCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *CreateConfigureKubernetesOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - Debug: o.CommonOptions.Verbose, - Namespace: o.CommonOptions.Namespace, - } - - if o.localStorageChanged { - deploy := o.LocalStorage - arg.DeployLocalStorage = &deploy - } - - return kubernetes.CreateConfigureKubernetes(arg) -} - -func (o *CreateConfigureKubernetesOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().BoolVarP(&o.LocalStorage, "with-local-storage", "", false, "Deploy a local PV provisioner") -} diff --git a/cmd/kk/cmd/create/phase/etcd.go b/cmd/kk/cmd/create/phase/etcd.go deleted file mode 100755 index 7248164b..00000000 --- a/cmd/kk/cmd/create/phase/etcd.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/etcd" -) - -type CreateEtcdOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string -} - -func NewCreateEtcdOptions() *CreateEtcdOptions { - return &CreateEtcdOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreateEtcd creates a new install etcd command -func NewCmdCreateEtcd() *cobra.Command { - o := NewCreateEtcdOptions() - cmd := &cobra.Command{ - Use: "etcd", - Short: "Install the ETCD cluster on the master", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *CreateEtcdOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - } - return etcd.CreateEtcd(arg) -} - -func (o *CreateEtcdOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") -} diff --git a/cmd/kk/cmd/create/phase/images.go b/cmd/kk/cmd/create/phase/images.go deleted file mode 100755 index 01c09685..00000000 --- a/cmd/kk/cmd/create/phase/images.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/images" -) - -type CreateImagesOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string - ContainerManager string -} - -func NewCreateImagesOptions() *CreateImagesOptions { - return &CreateImagesOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreateImages creates a new Images command -func NewCmdCreateImages() *cobra.Command { - o := NewCreateImagesOptions() - cmd := &cobra.Command{ - Use: "images", - Short: "Down the container and pull the images before creating your cluster", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Validate(cmd, args)) - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := k8sCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *CreateImagesOptions) Validate(_ *cobra.Command, _ []string) error { - switch o.ContainerManager { - case common.Docker, common.Containerd, common.Crio, common.Isula: - default: - return fmt.Errorf("unsupport container runtime [%s]", o.ContainerManager) - } - return nil -} - -func (o *CreateImagesOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - ContainerManager: o.ContainerManager, - Debug: o.CommonOptions.Verbose, - } - return images.CreateImages(arg) -} - -func (o *CreateImagesOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().StringVarP(&o.ContainerManager, "container-manager", "", "docker", "Container runtime: docker, crio, containerd and isula.") -} diff --git a/cmd/kk/cmd/create/phase/init.go b/cmd/kk/cmd/create/phase/init.go deleted file mode 100644 index 60f5c83b..00000000 --- a/cmd/kk/cmd/create/phase/init.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/kubernetes" -) - -type CreateInitClusterOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string -} - -func NewCreateInitClusterOptions() *CreateInitClusterOptions { - return &CreateInitClusterOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdUpgrade creates a new upgrade command -func NewCmdCreateInitCluster() *cobra.Command { - o := NewCreateInitClusterOptions() - cmd := &cobra.Command{ - Use: "init", - Short: "Init the k8s cluster", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := k8sCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *CreateInitClusterOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - Debug: o.CommonOptions.Verbose, - Namespace: o.CommonOptions.Namespace, - } - - return kubernetes.CreateInitCluster(arg) -} - -func (o *CreateInitClusterOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") -} diff --git a/cmd/kk/cmd/create/phase/join.go b/cmd/kk/cmd/create/phase/join.go deleted file mode 100644 index 31a442ac..00000000 --- a/cmd/kk/cmd/create/phase/join.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/kubernetes" -) - -type CreateJoinNodesOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string -} - -func NewCreateJoinNodesOptions() *CreateJoinNodesOptions { - return &CreateJoinNodesOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreateJoinNodes creates a new join nodes phase command -func NewCmdCreateJoinNodes() *cobra.Command { - o := NewCreateJoinNodesOptions() - cmd := &cobra.Command{ - Use: "join", - Short: "Join the control-plane nodes and worker nodes in the k8s cluster", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := k8sCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *CreateJoinNodesOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - Debug: o.CommonOptions.Verbose, - Namespace: o.CommonOptions.Namespace, - } - - return kubernetes.CreateJoinNodes(arg) -} - -func (o *CreateJoinNodesOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") -} diff --git a/cmd/kk/cmd/create/phase/kubesphere.go b/cmd/kk/cmd/create/phase/kubesphere.go deleted file mode 100755 index 39f09d32..00000000 --- a/cmd/kk/cmd/create/phase/kubesphere.go +++ /dev/null @@ -1,101 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - "time" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - alpha "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/kubesphere" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -type CreateKubeSphereOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - EnableKubeSphere bool - KubeSphere string -} - -func NewCreateKubeSphereOptions() *CreateKubeSphereOptions { - return &CreateKubeSphereOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdCreateKubeSphere creates a new CreateKubeSphere command -func NewCmdCreateKubeSphere() *cobra.Command { - o := NewCreateKubeSphereOptions() - cmd := &cobra.Command{ - Use: "kubesphere", - Short: "Install the kubesphere with the input version", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := ksCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *CreateKubeSphereOptions) Complete(cmd *cobra.Command, args []string) error { - var ksVersion string - if o.EnableKubeSphere && len(args) > 0 { - ksVersion = args[0] - } else { - ksVersion = kubesphere.Latest().Version - } - o.KubeSphere = ksVersion - return nil -} - -func (o *CreateKubeSphereOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KsEnable: o.EnableKubeSphere, - KsVersion: o.KubeSphere, - SkipConfirmCheck: o.CommonOptions.SkipConfirmCheck, - Debug: o.CommonOptions.Verbose, - } - return alpha.CreateKubeSphere(arg) -} - -func (o *CreateKubeSphereOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().BoolVarP(&o.EnableKubeSphere, "with-kubesphere", "", false, fmt.Sprintf("Deploy a specific version of kubesphere (default %s)", kubesphere.Latest().Version)) -} - -func ksCompletionSetting(cmd *cobra.Command) (err error) { - cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ( - strings []string, directive cobra.ShellCompDirective) { - versionArray := kubesphere.VersionsStringArr() - versionArray = append(versionArray, time.Now().Add(-time.Hour*24).Format("nightly-20060102")) - return versionArray, cobra.ShellCompDirectiveNoFileComp - } - - return -} diff --git a/cmd/kk/cmd/create/phase/os.go b/cmd/kk/cmd/create/phase/os.go deleted file mode 100755 index fc7eb76d..00000000 --- a/cmd/kk/cmd/create/phase/os.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/os" -) - -type ConfigOSOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - InstallPackages bool -} - -func NewConfigOSOptions() *ConfigOSOptions { - return &ConfigOSOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdConfigOS creates a new init os command -func NewCmdConfigOS() *cobra.Command { - o := NewConfigOSOptions() - cmd := &cobra.Command{ - Use: "os", - Short: "Init the os configure", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *ConfigOSOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - InstallPackages: o.InstallPackages, - } - return os.ConfigOS(arg) -} - -func (o *ConfigOSOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().BoolVarP(&o.InstallPackages, "with-packages", "", false, "install operation system packages by artifact") -} diff --git a/cmd/kk/cmd/create/phase/phase.go b/cmd/kk/cmd/create/phase/phase.go deleted file mode 100755 index 655fddea..00000000 --- a/cmd/kk/cmd/create/phase/phase.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "github.com/spf13/cobra" -) - -func NewPhaseCommand() *cobra.Command { - cmds := &cobra.Command{ - Use: "phase", - Short: "KubeKey create phase", - Long: `This is the create phase run cmd`, - } - cmds.AddCommand(NewCmdCreateBinary()) - cmds.AddCommand(NewCmdConfigOS()) - cmds.AddCommand(NewCmdCreateImages()) - cmds.AddCommand(NewCmdCreateEtcd()) - cmds.AddCommand(NewCmdCreateInitCluster()) - cmds.AddCommand(NewCmdCreateJoinNodes()) - cmds.AddCommand(NewCmdCreateConfigureKubernetes()) - cmds.AddCommand(NewCmdCreateKubeSphere()) - - return cmds -} diff --git a/cmd/kk/cmd/delete/delete.go b/cmd/kk/cmd/delete/delete.go deleted file mode 100644 index 68777db4..00000000 --- a/cmd/kk/cmd/delete/delete.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package delete - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type DeleteOptions struct { - CommonOptions *options.CommonOptions -} - -func NewDeleteOptions() *DeleteOptions { - return &DeleteOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdDelete creates a new delete command -func NewCmdDelete() *cobra.Command { - o := NewDeleteOptions() - cmd := &cobra.Command{ - Use: "delete", - Short: "Delete node or cluster", - } - - o.CommonOptions.AddCommonFlag(cmd) - - cmd.AddCommand(NewCmdDeleteCluster()) - cmd.AddCommand(NewCmdDeleteNode()) - return cmd -} diff --git a/cmd/kk/cmd/delete/delete_cluster.go b/cmd/kk/cmd/delete/delete_cluster.go deleted file mode 100644 index 19fac4d2..00000000 --- a/cmd/kk/cmd/delete/delete_cluster.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package delete - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type DeleteClusterOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string - DeleteCRI bool -} - -func NewDeleteClusterOptions() *DeleteClusterOptions { - return &DeleteClusterOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdDeleteCluster creates a new delete cluster command -func NewCmdDeleteCluster() *cobra.Command { - o := NewDeleteClusterOptions() - cmd := &cobra.Command{ - Use: "cluster", - Short: "Delete a cluster", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *DeleteClusterOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - KubernetesVersion: o.Kubernetes, - DeleteCRI: o.DeleteCRI, - SkipConfirmCheck: o.CommonOptions.SkipConfirmCheck, - } - return pipelines.DeleteCluster(arg) -} - -func (o *DeleteClusterOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().BoolVarP(&o.DeleteCRI, "all", "A", false, "Delete total cri conficutation and data directories") -} diff --git a/cmd/kk/cmd/delete/delete_node.go b/cmd/kk/cmd/delete/delete_node.go deleted file mode 100644 index 115db96e..00000000 --- a/cmd/kk/cmd/delete/delete_node.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package delete - -import ( - "strings" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type DeleteNodeOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - nodeName string -} - -func NewDeleteNodeOptions() *DeleteNodeOptions { - return &DeleteNodeOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdDeleteNode creates a new delete node command -func NewCmdDeleteNode() *cobra.Command { - o := NewDeleteNodeOptions() - cmd := &cobra.Command{ - Use: "node", - Short: "delete a node", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Validate()) - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *DeleteNodeOptions) Complete(cmd *cobra.Command, args []string) error { - o.nodeName = strings.Join(args, "") - return nil -} - -func (o *DeleteNodeOptions) Validate() error { - if o.nodeName == "" { - return errors.New("node name can not be empty") - } - return nil -} - -func (o *DeleteNodeOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - NodeName: o.nodeName, - SkipConfirmCheck: o.CommonOptions.SkipConfirmCheck, - } - return pipelines.DeleteNode(arg) -} - -func (o *DeleteNodeOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - -} diff --git a/cmd/kk/cmd/init/init.go b/cmd/kk/cmd/init/init.go deleted file mode 100644 index cea0d54e..00000000 --- a/cmd/kk/cmd/init/init.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package init - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type InitOptions struct { - CommonOptions *options.CommonOptions -} - -func NewInitOptions() *InitOptions { - return &InitOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdInit create a new init command -func NewCmdInit() *cobra.Command { - o := NewInitOptions() - cmd := &cobra.Command{ - Use: "init", - Short: "Initializes the installation environment", - } - o.CommonOptions.AddCommonFlag(cmd) - cmd.AddCommand(NewCmdInitOs()) - cmd.AddCommand(NewCmdInitRegistry()) - return cmd -} diff --git a/cmd/kk/cmd/init/init_os.go b/cmd/kk/cmd/init/init_os.go deleted file mode 100644 index 1b23bc8c..00000000 --- a/cmd/kk/cmd/init/init_os.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package init - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type InitOsOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Artifact string -} - -func NewInitOsOptions() *InitOsOptions { - return &InitOsOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdInitOs creates a new init os command -func NewCmdInitOs() *cobra.Command { - o := NewInitOsOptions() - cmd := &cobra.Command{ - Use: "os", - Short: "Init operating system", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *InitOsOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - Artifact: o.Artifact, - } - return pipelines.InitDependencies(arg) -} - -func (o *InitOsOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact") -} diff --git a/cmd/kk/cmd/init/init_registry.go b/cmd/kk/cmd/init/init_registry.go deleted file mode 100644 index bfae0df9..00000000 --- a/cmd/kk/cmd/init/init_registry.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package init - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" -) - -type InitRegistryOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - DownloadCmd string - Artifact string -} - -func NewInitRegistryOptions() *InitRegistryOptions { - return &InitRegistryOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdInitRegistry creates a new init os command -func NewCmdInitRegistry() *cobra.Command { - o := NewInitRegistryOptions() - cmd := &cobra.Command{ - Use: "registry", - Short: "Init a local image registry", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - return cmd -} - -func (o *InitRegistryOptions) Complete(_ *cobra.Command, _ []string) error { - return nil -} - -func (o *InitRegistryOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - Debug: o.CommonOptions.Verbose, - Artifact: o.Artifact, - } - return pipelines.InitRegistry(arg, o.DownloadCmd) -} - -func (o *InitRegistryOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", - `The user defined command to download the necessary files. The first param '%s' is output path, the second param '%s', is the URL`) - cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact") -} diff --git a/cmd/kk/cmd/options/common_options.go b/cmd/kk/cmd/options/common_options.go deleted file mode 100644 index 0badd11b..00000000 --- a/cmd/kk/cmd/options/common_options.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/cobra" -) - -type CommonOptions struct { - Verbose bool - SkipConfirmCheck bool - IgnoreErr bool - Namespace string -} - -func NewCommonOptions() *CommonOptions { - return &CommonOptions{} -} - -func (o *CommonOptions) AddCommonFlag(cmd *cobra.Command) { - cmd.Flags().BoolVar(&o.Verbose, "debug", false, "Print detailed information") - cmd.Flags().BoolVarP(&o.SkipConfirmCheck, "yes", "y", false, "Skip confirm check") - cmd.Flags().BoolVar(&o.IgnoreErr, "ignore-err", false, "Ignore the error message, remove the host which reported error and force to continue") - cmd.Flags().StringVar(&o.Namespace, "namespace", "kubekey-system", "KubeKey namespace to use") -} diff --git a/cmd/kk/cmd/options/io_options.go b/cmd/kk/cmd/options/io_options.go deleted file mode 100644 index 02532fd2..00000000 --- a/cmd/kk/cmd/options/io_options.go +++ /dev/null @@ -1,30 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package options - -import "io" - -// IOStreams provides the standard names for iostreams. This is useful for embedding and for unit testing. -// Inconsistent and different names make it hard to read and review code -type IOStreams struct { - // In think, os.Stdin - In io.Reader - // Out think, os.Stdout - Out io.Writer - // ErrOut think, os.Stderr - ErrOut io.Writer -} diff --git a/cmd/kk/cmd/plugin/list.go b/cmd/kk/cmd/plugin/list.go deleted file mode 100644 index 0d7bd7b0..00000000 --- a/cmd/kk/cmd/plugin/list.go +++ /dev/null @@ -1,237 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package plugin - -import ( - "bytes" - "fmt" - "os" - "path/filepath" - "runtime" - "strings" - - "github.com/spf13/cobra" - cmdutil "k8s.io/kubectl/pkg/cmd/util" - "k8s.io/kubectl/pkg/util/i18n" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -type PluginListOptions struct { - Verifier PathVerifier - NameOnly bool - PluginPaths []string - options.IOStreams -} - -// NewCmdPluginList provides a way to list all plugin executables visible to kubectl -func NewCmdPluginList(streams options.IOStreams) *cobra.Command { - o := &PluginListOptions{ - IOStreams: streams, - } - - cmd := &cobra.Command{ - Use: "list", - Short: i18n.T("List all visible plugin executables on a user's PATH"), - Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(o.Complete(cmd)) - cmdutil.CheckErr(o.Run()) - }, - } - - cmd.Flags().BoolVar(&o.NameOnly, "name-only", o.NameOnly, "If true, display only the binary name of each plugin, rather than its full path") - return cmd -} - -func (o *PluginListOptions) Complete(cmd *cobra.Command) error { - o.Verifier = &CommandOverrideVerifier{ - root: cmd.Root(), - seenPlugins: make(map[string]string), - } - - o.PluginPaths = filepath.SplitList(os.Getenv("PATH")) - return nil -} - -func (o *PluginListOptions) Run() error { - pluginsFound := false - isFirstFile := true - pluginErrors := []error{} - pluginWarnings := 0 - - for _, dir := range uniquePathsList(o.PluginPaths) { - if len(strings.TrimSpace(dir)) == 0 { - continue - } - - files, err := os.ReadDir(dir) - if err != nil { - if _, ok := err.(*os.PathError); ok { - fmt.Fprintf(o.ErrOut, "Unable to read directory %q from your PATH: %v. Skipping...\n", dir, err) - continue - } - - pluginErrors = append(pluginErrors, fmt.Errorf("error: unable to read directory %q in your PATH: %v", dir, err)) - continue - } - - for _, f := range files { - if f.IsDir() { - continue - } - if !hasValidPrefix(f.Name(), ValidPluginFilenamePrefixes) { - continue - } - - if isFirstFile { - fmt.Fprintf(o.Out, "The following compatible plugins are available:\n\n") - pluginsFound = true - isFirstFile = false - } - - pluginPath := f.Name() - if !o.NameOnly { - pluginPath = filepath.Join(dir, pluginPath) - } - - fmt.Fprintf(o.Out, "%s\n", pluginPath) - if errs := o.Verifier.Verify(filepath.Join(dir, f.Name())); len(errs) != 0 { - for _, err := range errs { - fmt.Fprintf(o.ErrOut, " - %s\n", err) - pluginWarnings++ - } - } - } - } - - if !pluginsFound { - pluginErrors = append(pluginErrors, fmt.Errorf("error: unable to find any kubectl plugins in your PATH")) - } - - if pluginWarnings > 0 { - if pluginWarnings == 1 { - pluginErrors = append(pluginErrors, fmt.Errorf("error: one plugin warning was found")) - } else { - pluginErrors = append(pluginErrors, fmt.Errorf("error: %v plugin warnings were found", pluginWarnings)) - } - } - if len(pluginErrors) > 0 { - errs := bytes.NewBuffer(nil) - for _, e := range pluginErrors { - fmt.Fprintln(errs, e) - } - return fmt.Errorf("%s", errs.String()) - } - - return nil -} - -type PathVerifier interface { - // Verify determines if a given path is valid - Verify(path string) []error -} - -type CommandOverrideVerifier struct { - root *cobra.Command - seenPlugins map[string]string -} - -// Verify implements PathVerifier and determines if a given path -// is valid depending on whether or not it overwrites an existing -// kubectl command path, or a previously seen plugin. -func (v *CommandOverrideVerifier) Verify(path string) []error { - if v.root == nil { - return []error{fmt.Errorf("unable to verify path with nil root")} - } - - // extract the plugin binary name - segs := strings.Split(path, "/") - binName := segs[len(segs)-1] - - cmdPath := strings.Split(binName, "-") - if len(cmdPath) > 1 { - // the first argument is always "kubectl" for a plugin binary - cmdPath = cmdPath[1:] - } - - errors := []error{} - - if isExec, err := isExecutable(path); err == nil && !isExec { - errors = append(errors, fmt.Errorf("warning: %s identified as a kubectl plugin, but it is not executable", path)) - } else if err != nil { - errors = append(errors, fmt.Errorf("error: unable to identify %s as an executable file: %v", path, err)) - } - - if existingPath, ok := v.seenPlugins[binName]; ok { - errors = append(errors, fmt.Errorf("warning: %s is overshadowed by a similarly named plugin: %s", path, existingPath)) - } else { - v.seenPlugins[binName] = path - } - - if cmd, _, err := v.root.Find(cmdPath); err == nil { - errors = append(errors, fmt.Errorf("warning: %s overwrites existing command: %q", binName, cmd.CommandPath())) - } - - return errors -} - -func isExecutable(fullPath string) (bool, error) { - info, err := os.Stat(fullPath) - if err != nil { - return false, err - } - - if runtime.GOOS == "windows" { - fileExt := strings.ToLower(filepath.Ext(fullPath)) - - switch fileExt { - case ".bat", ".cmd", ".com", ".exe", ".ps1": - return true, nil - } - return false, nil - } - - if m := info.Mode(); !m.IsDir() && m&0111 != 0 { - return true, nil - } - - return false, nil -} - -// uniquePathsList deduplicates a given slice of strings without -// sorting or otherwise altering its order in any way. -func uniquePathsList(paths []string) []string { - seen := map[string]bool{} - newPaths := []string{} - for _, p := range paths { - if seen[p] { - continue - } - seen[p] = true - newPaths = append(newPaths, p) - } - return newPaths -} - -func hasValidPrefix(filepath string, validPrefixes []string) bool { - for _, prefix := range validPrefixes { - if strings.HasPrefix(filepath, prefix+"-") { - return true - } - } - return false -} diff --git a/cmd/kk/cmd/plugin/plugin.go b/cmd/kk/cmd/plugin/plugin.go deleted file mode 100644 index cabf9dde..00000000 --- a/cmd/kk/cmd/plugin/plugin.go +++ /dev/null @@ -1,38 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package plugin - -import ( - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" -) - -var ( - ValidPluginFilenamePrefixes = []string{"kk"} -) - -func NewCmdPlugin(streams options.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "plugin [flags]", - DisableFlagsInUseLine: true, - Short: "Provides utilities for interacting with plugins", - } - - cmd.AddCommand(NewCmdPluginList(streams)) - return cmd -} diff --git a/cmd/kk/cmd/root.go b/cmd/kk/cmd/root.go deleted file mode 100755 index 3ca697a9..00000000 --- a/cmd/kk/cmd/root.go +++ /dev/null @@ -1,230 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "fmt" - "os" - "os/exec" - "runtime" - "strings" - "syscall" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/add" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/alpha" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/cert" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/completion" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/create" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/delete" - initOs "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/init" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/plugin" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/upgrade" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/version" -) - -type KubeKeyOptions struct { - PluginHandler PluginHandler - Arguments []string - - options.IOStreams -} - -func NewDefaultKubeKeyCommand() *cobra.Command { - return NewDefaultKubeKeyCommandWithArgs(KubeKeyOptions{ - PluginHandler: NewDefaultPluginHandler(plugin.ValidPluginFilenamePrefixes), - Arguments: os.Args, - - IOStreams: options.IOStreams{In: os.Stdin, Out: os.Stdout, ErrOut: os.Stderr}, - }) -} - -func NewDefaultKubeKeyCommandWithArgs(o KubeKeyOptions) *cobra.Command { - cmd := NewKubeKeyCommand(o) - - if o.PluginHandler == nil { - return cmd - } - - if len(o.Arguments) > 1 { - cmdPathPieces := o.Arguments[1:] - - // only look for suitable extension executables if - // the specified command does not already exist - if _, _, err := cmd.Find(cmdPathPieces); err != nil { - // Also check the commands that will be added by Cobra. - // These commands are only added once rootCmd.Execute() is called, so we - // need to check them explicitly here. - var cmdName string // first "non-flag" arguments - for _, arg := range cmdPathPieces { - if !strings.HasPrefix(arg, "-") { - cmdName = arg - break - } - } - - switch cmdName { - case "help", cobra.ShellCompRequestCmd, cobra.ShellCompNoDescRequestCmd: - // Don't search for a plugin - default: - if err := HandlePluginCommand(o.PluginHandler, cmdPathPieces); err != nil { - os.Exit(1) - } - } - } - } - - return cmd -} - -// NewKubeKeyCommand creates a new kubekey root command -func NewKubeKeyCommand(o KubeKeyOptions) *cobra.Command { - cmds := &cobra.Command{ - Use: "kk", - Short: "Kubernetes/KubeSphere Deploy Tool", - Long: `Deploy a Kubernetes or KubeSphere cluster efficiently, flexibly and easily. There are three scenarios to use KubeKey. -1. Install Kubernetes only -2. Install Kubernetes and KubeSphere together in one command -3. Install Kubernetes first, then deploy KubeSphere on it using https://github.com/kubesphere/ks-installer`, - } - - cmds.AddCommand(initOs.NewCmdInit()) - - cmds.AddCommand(alpha.NewAlphaCmd()) - - cmds.AddCommand(create.NewCmdCreate()) - cmds.AddCommand(delete.NewCmdDelete()) - cmds.AddCommand(add.NewCmdAdd()) - cmds.AddCommand(upgrade.NewCmdUpgrade()) - cmds.AddCommand(cert.NewCmdCerts()) - cmds.AddCommand(artifact.NewCmdArtifact()) - - cmds.AddCommand(plugin.NewCmdPlugin(o.IOStreams)) - - cmds.AddCommand(completion.NewCmdCompletion()) - cmds.AddCommand(version.NewCmdVersion()) - - return cmds -} - -// PluginHandler is capable of parsing command line arguments -// and performing executable filename lookups to search -// for valid plugin files, and execute found plugins. -type PluginHandler interface { - // exists at the given filename, or a boolean false. - // Lookup will iterate over a list of given prefixes - // in order to recognize valid plugin filenames. - // The first filepath to match a prefix is returned. - Lookup(filename string) (string, bool) - // Execute receives an executable's filepath, a slice - // of arguments, and a slice of environment variables - // to relay to the executable. - Execute(executablePath string, cmdArgs, environment []string) error -} - -// DefaultPluginHandler implements PluginHandler -type DefaultPluginHandler struct { - ValidPrefixes []string -} - -// NewDefaultPluginHandler instantiates the DefaultPluginHandler with a list of -// given filename prefixes used to identify valid plugin filenames. -func NewDefaultPluginHandler(validPrefixes []string) *DefaultPluginHandler { - return &DefaultPluginHandler{ - ValidPrefixes: validPrefixes, - } -} - -// Lookup implements PluginHandler -func (h *DefaultPluginHandler) Lookup(filename string) (string, bool) { - for _, prefix := range h.ValidPrefixes { - path, err := exec.LookPath(fmt.Sprintf("%s-%s", prefix, filename)) - if err != nil || len(path) == 0 { - continue - } - return path, true - } - - return "", false -} - -// Execute implements PluginHandler -func (h *DefaultPluginHandler) Execute(executablePath string, cmdArgs, environment []string) error { - - // Windows does not support exec syscall. - if runtime.GOOS == "windows" { - cmd := exec.Command(executablePath, cmdArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdin - cmd.Env = environment - err := cmd.Run() - if err == nil { - os.Exit(0) - } - return err - } - - // invoke cmd binary relaying the environment and args given - // append executablePath to cmdArgs, as execve will make first argument the "binary name". - return syscall.Exec(executablePath, append([]string{executablePath}, cmdArgs...), environment) -} - -// HandlePluginCommand receives a pluginHandler and command-line arguments and attempts to find -// a plugin executable on the PATH that satisfies the given arguments. -func HandlePluginCommand(pluginHandler PluginHandler, cmdArgs []string) error { - var remainingArgs []string // all "non-flag" arguments - for _, arg := range cmdArgs { - if strings.HasPrefix(arg, "-") { - break - } - remainingArgs = append(remainingArgs, strings.Replace(arg, "-", "_", -1)) - } - - if len(remainingArgs) == 0 { - // the length of cmdArgs is at least 1 - return fmt.Errorf("flags cannot be placed before plugin name: %s", cmdArgs[0]) - } - - foundBinaryPath := "" - - // attempt to find binary, starting at longest possible name with given cmdArgs - for len(remainingArgs) > 0 { - path, found := pluginHandler.Lookup(strings.Join(remainingArgs, "-")) - if !found { - remainingArgs = remainingArgs[:len(remainingArgs)-1] - continue - } - - foundBinaryPath = path - break - } - - if len(foundBinaryPath) == 0 { - return nil - } - - // invoke cmd binary relaying the current environment and args given - if err := pluginHandler.Execute(foundBinaryPath, cmdArgs[len(remainingArgs):], os.Environ()); err != nil { - return err - } - - return nil -} diff --git a/cmd/kk/cmd/upgrade/phase/binary.go b/cmd/kk/cmd/upgrade/phase/binary.go deleted file mode 100755 index 2e2bdc62..00000000 --- a/cmd/kk/cmd/upgrade/phase/binary.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/binary" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes" -) - -type UpgradeBinaryOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string - DownloadCmd string -} - -func NewUpgradeBinaryOptions() *UpgradeBinaryOptions { - return &UpgradeBinaryOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdUpgradeBinary creates a new artifact import command -func NewCmdUpgradeBinary() *cobra.Command { - o := NewUpgradeBinaryOptions() - cmd := &cobra.Command{ - Use: "binary", - Short: "Download the binary and synchronize kubernetes binaries", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - if err := k8sCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *UpgradeBinaryOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - Debug: o.CommonOptions.Verbose, - } - return binary.UpgradeBinary(arg, o.DownloadCmd) -} - -func (o *UpgradeBinaryOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", - `The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`) - -} - -func k8sCompletionSetting(cmd *cobra.Command) (err error) { - err = cmd.RegisterFlagCompletionFunc("with-kubernetes", func(cmd *cobra.Command, args []string, toComplete string) ( - strings []string, directive cobra.ShellCompDirective) { - return kubernetes.SupportedK8sVersionList(), cobra.ShellCompDirectiveNoFileComp - }) - return -} diff --git a/cmd/kk/cmd/upgrade/phase/images.go b/cmd/kk/cmd/upgrade/phase/images.go deleted file mode 100755 index 86bd97a4..00000000 --- a/cmd/kk/cmd/upgrade/phase/images.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/images" -) - -type UpgradeImagesOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string -} - -func NewUpgradeImagesOptions() *UpgradeImagesOptions { - return &UpgradeImagesOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdUpgrade creates a new upgrade command -func NewCmdUpgradeImages() *cobra.Command { - o := NewUpgradeImagesOptions() - cmd := &cobra.Command{ - Use: "images", - Short: "Pull the images before upgrading your cluster", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := k8sCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *UpgradeImagesOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - Debug: o.CommonOptions.Verbose, - } - return images.UpgradeImages(arg) -} - -func (o *UpgradeImagesOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") -} diff --git a/cmd/kk/cmd/upgrade/phase/kubesphere.go b/cmd/kk/cmd/upgrade/phase/kubesphere.go deleted file mode 100755 index 003b2307..00000000 --- a/cmd/kk/cmd/upgrade/phase/kubesphere.go +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - "time" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - alpha "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/kubesphere" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -type UpgradeKubeSphereOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - EnableKubeSphere bool - KubeSphere string -} - -func NewUpgradeKubeSphereOptions() *UpgradeKubeSphereOptions { - return &UpgradeKubeSphereOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdUpgradeKubeSphere creates a new UpgradeKubeSphere command -func NewCmdUpgradeKubeSphere() *cobra.Command { - o := NewUpgradeKubeSphereOptions() - cmd := &cobra.Command{ - Use: "kubesphere", - Short: "Upgrade your kubesphere to a newer version with this command", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := ksCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *UpgradeKubeSphereOptions) Complete(cmd *cobra.Command, args []string) error { - var ksVersion string - if o.EnableKubeSphere && len(args) > 0 { - ksVersion = args[0] - } else { - ksVersion = kubesphere.Latest().Version - } - o.KubeSphere = ksVersion - return nil -} - -func (o *UpgradeKubeSphereOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KsEnable: o.EnableKubeSphere, - KsVersion: o.KubeSphere, - SkipConfirmCheck: o.CommonOptions.SkipConfirmCheck, - Debug: o.CommonOptions.Verbose, - } - return alpha.UpgradeKubeSphere(arg) -} - -func (o *UpgradeKubeSphereOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().BoolVarP(&o.EnableKubeSphere, "with-kubesphere", "", false, fmt.Sprintf("Deploy a specific version of kubesphere (default %s)", kubesphere.Latest().Version)) -} - -func ksCompletionSetting(cmd *cobra.Command) (err error) { - cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ( - strings []string, directive cobra.ShellCompDirective) { - versionArray := kubesphere.VersionsStringArr() - versionArray = append(versionArray, time.Now().Add(-time.Hour*24).Format("nightly-20060102")) - return versionArray, cobra.ShellCompDirectiveNoFileComp - } - return -} diff --git a/cmd/kk/cmd/upgrade/phase/nodes.go b/cmd/kk/cmd/upgrade/phase/nodes.go deleted file mode 100755 index 83b0eaa2..00000000 --- a/cmd/kk/cmd/upgrade/phase/nodes.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/nodes" -) - -type UpgradeNodesOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string -} - -func NewUpgradeNodesOptions() *UpgradeNodesOptions { - return &UpgradeNodesOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdUpgrade creates a new upgrade command -func NewCmdUpgradeNodes() *cobra.Command { - o := NewUpgradeNodesOptions() - cmd := &cobra.Command{ - Use: "nodes", - Short: "Upgrade cluster on master nodes and worker nodes to the version you input", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := k8sCompletionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *UpgradeNodesOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - Debug: o.CommonOptions.Verbose, - } - return nodes.UpgradeNodes(arg) -} - -func (o *UpgradeNodesOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") -} diff --git a/cmd/kk/cmd/upgrade/phase/phase.go b/cmd/kk/cmd/upgrade/phase/phase.go deleted file mode 100755 index f17d1c7b..00000000 --- a/cmd/kk/cmd/upgrade/phase/phase.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phase - -import ( - "github.com/spf13/cobra" -) - -func NewPhaseCommand() *cobra.Command { - cmds := &cobra.Command{ - Use: "phase", - Short: "KubeKey upgrade phase", - Long: `This is the upgrade phase run cmd`, - } - - cmds.AddCommand(NewCmdUpgradeBinary()) - cmds.AddCommand(NewCmdUpgradeImages()) - cmds.AddCommand(NewCmdUpgradeNodes()) - cmds.AddCommand(NewCmdUpgradeKubeSphere()) - - return cmds -} diff --git a/cmd/kk/cmd/upgrade/upgrade.go b/cmd/kk/cmd/upgrade/upgrade.go deleted file mode 100644 index 4e9f6038..00000000 --- a/cmd/kk/cmd/upgrade/upgrade.go +++ /dev/null @@ -1,118 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "fmt" - "time" - - "github.com/spf13/cobra" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/options" - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/pipelines" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -type UpgradeOptions struct { - CommonOptions *options.CommonOptions - ClusterCfgFile string - Kubernetes string - EnableKubeSphere bool - KubeSphere string - SkipPullImages bool - DownloadCmd string - Artifact string -} - -func NewUpgradeOptions() *UpgradeOptions { - return &UpgradeOptions{ - CommonOptions: options.NewCommonOptions(), - } -} - -// NewCmdUpgrade creates a new upgrade command -func NewCmdUpgrade() *cobra.Command { - o := NewUpgradeOptions() - cmd := &cobra.Command{ - Use: "upgrade", - Short: "Upgrade your cluster smoothly to a newer version with this command", - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(cmd, args)) - util.CheckErr(o.Run()) - }, - } - o.CommonOptions.AddCommonFlag(cmd) - o.AddFlags(cmd) - - if err := completionSetting(cmd); err != nil { - panic(fmt.Sprintf("Got error with the completion setting")) - } - return cmd -} - -func (o *UpgradeOptions) Complete(cmd *cobra.Command, args []string) error { - var ksVersion string - if o.EnableKubeSphere && len(args) > 0 { - ksVersion = args[0] - } else { - ksVersion = kubesphere.Latest().Version - } - o.KubeSphere = ksVersion - return nil -} - -func (o *UpgradeOptions) Run() error { - arg := common.Argument{ - FilePath: o.ClusterCfgFile, - KubernetesVersion: o.Kubernetes, - KsEnable: o.EnableKubeSphere, - KsVersion: o.KubeSphere, - SkipPullImages: o.SkipPullImages, - Debug: o.CommonOptions.Verbose, - SkipConfirmCheck: o.CommonOptions.SkipConfirmCheck, - Artifact: o.Artifact, - } - return pipelines.UpgradeCluster(arg, o.DownloadCmd) -} - -func (o *UpgradeOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.ClusterCfgFile, "filename", "f", "", "Path to a configuration file") - cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes") - cmd.Flags().BoolVarP(&o.EnableKubeSphere, "with-kubesphere", "", false, fmt.Sprintf("Deploy a specific version of kubesphere (default %s)", kubesphere.Latest().Version)) - cmd.Flags().BoolVarP(&o.SkipPullImages, "skip-pull-images", "", false, "Skip pre pull images") - cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s", - `The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`) - cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact") -} - -func completionSetting(cmd *cobra.Command) (err error) { - cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ( - strings []string, directive cobra.ShellCompDirective) { - versionArray := kubesphere.VersionsStringArr() - versionArray = append(versionArray, time.Now().Add(-time.Hour*24).Format("nightly-20060102")) - return versionArray, cobra.ShellCompDirectiveNoFileComp - } - - err = cmd.RegisterFlagCompletionFunc("with-kubernetes", func(cmd *cobra.Command, args []string, toComplete string) ( - strings []string, directive cobra.ShellCompDirective) { - return kubernetes.SupportedK8sVersionList(), cobra.ShellCompDirectiveNoFileComp - }) - return -} diff --git a/cmd/kk/cmd/util/helpers.go b/cmd/kk/cmd/util/helpers.go deleted file mode 100644 index 1f4bffc2..00000000 --- a/cmd/kk/cmd/util/helpers.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import ( - "fmt" - "os" - "strings" -) - -const DefaultErrorExitCode = 1 - -var ErrExit = fmt.Errorf("exit") - -func fatal(msg string, code int) { - if len(msg) > 0 { - // add newline if needed - if !strings.HasSuffix(msg, "\n") { - msg += "\n" - } - fmt.Fprint(os.Stderr, msg) - } - os.Exit(code) -} - -var fatalErrHandler = fatal - -func CheckErr(err error) { - checkErr(err, fatalErrHandler) -} - -func checkErr(err error, handleErr func(string, int)) { - if err == nil { - return - } - - switch { - case err == ErrExit: - handleErr("", DefaultErrorExitCode) - default: - msg := err.Error() - if !strings.HasPrefix(msg, "error: ") { - msg = fmt.Sprintf("error: %s", msg) - } - handleErr(msg, DefaultErrorExitCode) - } -} diff --git a/cmd/kk/cmd/version/version.go b/cmd/kk/cmd/version/version.go deleted file mode 100644 index 13b99070..00000000 --- a/cmd/kk/cmd/version/version.go +++ /dev/null @@ -1,102 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package version - -import ( - "encoding/json" - "fmt" - "io" - "strings" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - "gopkg.in/yaml.v3" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes" - "github.com/kubesphere/kubekey/v3/version" -) - -type Version struct { - KK *version.Info `json:"kk"` -} - -type VersionOptions struct { - Output string - ShowSupportedK8sVersionList bool -} - -func NewVersionOptions() *VersionOptions { - return &VersionOptions{} -} - -// NewCmdVersion creates a new version command -func NewCmdVersion() *cobra.Command { - o := NewVersionOptions() - - cmd := &cobra.Command{ - Use: "version", - Short: "print the client version information", - RunE: func(cmd *cobra.Command, _ []string) error { - if o.ShowSupportedK8sVersionList { - return printSupportedK8sVersionList(cmd.OutOrStdout()) - } - return o.Run() - }, - } - o.AddFlags(cmd) - return cmd -} - -func (o *VersionOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVarP(&o.Output, "output", "o", "", "Output format; available options are 'yaml', 'json' and 'short'") - cmd.Flags().BoolVarP(&o.ShowSupportedK8sVersionList, "show-supported-k8s", "", false, - `print the version of supported k8s`) -} - -func (o *VersionOptions) Run() error { - clientVersion := version.Get() - v := Version{ - KK: &clientVersion, - } - - switch o.Output { - case "": - fmt.Printf("kk version: %#v\n", v.KK) - case "short": - fmt.Printf("%s\n", v.KK.GitVersion) - case "yaml": - y, err := yaml.Marshal(&v) - if err != nil { - return err - } - fmt.Print(string(y)) - case "json": - y, err := json.MarshalIndent(&v, "", " ") - if err != nil { - return err - } - fmt.Println(string(y)) - default: - return errors.Errorf("invalid output format: %s", o.Output) - } - return nil -} - -func printSupportedK8sVersionList(output io.Writer) (err error) { - _, err = output.Write([]byte(fmt.Sprintln(strings.Join(kubernetes.SupportedK8sVersionList(), "\n")))) - return -} diff --git a/cmd/kk/main.go b/cmd/kk/main.go deleted file mode 100644 index 98f575e4..00000000 --- a/cmd/kk/main.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "os" - "os/exec" - - "github.com/kubesphere/kubekey/v3/cmd/kk/cmd" -) - -// Using a separate entry-point can reduce the size of the binary file -func main() { - c := cmd.NewDefaultKubeKeyCommand() - _ = exec.Command("/bin/bash", "-c", "ulimit -u 65535").Run() - _ = exec.Command("/bin/bash", "-c", "ulimit -n 65535").Run() - - // Execute adds all child commands to the root command and sets flags appropriately. - // This is called by main.main(). It only needs to happen once to the rootCmd. - if err := c.Execute(); err != nil { - //fmt.Println(err) - os.Exit(1) - } -} diff --git a/cmd/kk/pkg/addons/addons.go b/cmd/kk/pkg/addons/addons.go deleted file mode 100644 index 6d145e56..00000000 --- a/cmd/kk/pkg/addons/addons.go +++ /dev/null @@ -1,67 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package addons - -import ( - "net/url" - "os" - "path/filepath" - "strings" - - "github.com/pkg/errors" - "helm.sh/helm/v3/pkg/cli" - "helm.sh/helm/v3/pkg/getter" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" -) - -func InstallAddons(kubeConf *common.KubeConf, addon *kubekeyapiv1alpha2.Addon, kubeConfig string) error { - // install chart - if addon.Sources.Chart.Name != "" { - _ = os.Setenv("HELM_NAMESPACE", strings.TrimSpace(addon.Namespace)) - if err := InstallChart(kubeConf, addon, kubeConfig); err != nil { - return err - } - } - - // install yaml - if len(addon.Sources.Yaml.Path) != 0 { - var settings = cli.New() - p := getter.All(settings) - for _, yaml := range addon.Sources.Yaml.Path { - u, _ := url.Parse(yaml) - _, err := p.ByScheme(u.Scheme) - if err != nil { - fp, err := filepath.Abs(yaml) - if err != nil { - return errors.Wrap(err, "Failed to look up current directory") - } - yamlPaths := []string{fp} - if err := InstallYaml(yamlPaths, addon.Namespace, kubeConfig, kubeConf.Cluster.Kubernetes.Version); err != nil { - return err - } - } else { - yamlPaths := []string{yaml} - if err := InstallYaml(yamlPaths, addon.Namespace, kubeConfig, kubeConf.Cluster.Kubernetes.Version); err != nil { - return err - } - } - } - } - return nil -} diff --git a/cmd/kk/pkg/addons/charts.go b/cmd/kk/pkg/addons/charts.go deleted file mode 100644 index 176d3acf..00000000 --- a/cmd/kk/pkg/addons/charts.go +++ /dev/null @@ -1,246 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package addons - -import ( - "fmt" - "log" - "os" - "path/filepath" - "time" - - "github.com/pkg/errors" - "helm.sh/helm/v3/pkg/action" - "helm.sh/helm/v3/pkg/chart" - helmLoader "helm.sh/helm/v3/pkg/chart/loader" - "helm.sh/helm/v3/pkg/cli" - "helm.sh/helm/v3/pkg/cli/values" - "helm.sh/helm/v3/pkg/downloader" - "helm.sh/helm/v3/pkg/getter" - "helm.sh/helm/v3/pkg/release" - "helm.sh/helm/v3/pkg/storage/driver" - "k8s.io/client-go/util/homedir" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -func debug(format string, v ...interface{}) { - if false { - format = fmt.Sprintf("[debug] %s\n", format) - _ = log.Output(2, fmt.Sprintf(format, v...)) - } -} - -func InstallChart(kubeConf *common.KubeConf, addon *kubekeyapiv1alpha2.Addon, kubeConfig string) error { - actionConfig := new(action.Configuration) - var settings = cli.New() - helmDriver := os.Getenv("HELM_DRIVER") - settings.KubeConfig = kubeConfig - var namespace string - if addon.Namespace != "" { - namespace = addon.Namespace - } else { - namespace = "default" - } - - if err := actionConfig.Init(settings.RESTClientGetter(), namespace, helmDriver, debug); err != nil { - logger.Log.Fatal(err) - } - - valueOpts := &values.Options{} - if len(addon.Sources.Chart.Values) != 0 { - valueOpts.Values = addon.Sources.Chart.Values - } - if len(addon.Sources.Chart.ValuesFile) != 0 { - valueOpts.ValueFiles = []string{addon.Sources.Chart.ValuesFile} - } - - client := action.NewUpgrade(actionConfig) - - var chartName string - if addon.Sources.Chart.Name != "" { - if addon.Sources.Chart.Repo == "" && addon.Sources.Chart.Path != "" { - fmt.Println(addon.Sources.Chart.Repo) - chartName = filepath.Join(addon.Sources.Chart.Path, addon.Sources.Chart.Name) - } else { - chartName = addon.Sources.Chart.Name - } - } else { - logger.Log.Fatalln("No chart name is specified") - } - - args := []string{addon.Name, chartName} - - client.Install = true - client.Namespace = namespace - client.Timeout = 300 * time.Second - client.Keyring = defaultKeyring() - client.RepoURL = addon.Sources.Chart.Repo - client.Version = addon.Sources.Chart.Version - client.Wait = addon.Sources.Chart.Wait - //client.Force = true - - if client.Version == "" && client.Devel { - client.Version = ">0.0.0-0" - } - - if client.Install { - histClient := action.NewHistory(actionConfig) - histClient.Max = 1 - if _, err := histClient.Run(addon.Name); err == driver.ErrReleaseNotFound { - fmt.Printf("Release %q does not exist. Installing it now.\n", addon.Name) - instClient := action.NewInstall(actionConfig) - instClient.CreateNamespace = true - instClient.Namespace = client.Namespace - instClient.Timeout = client.Timeout - instClient.Keyring = client.Keyring - instClient.RepoURL = client.RepoURL - instClient.Version = client.Version - - r, err := runInstall(args, instClient, valueOpts, settings) - if err != nil { - return err - } - printReleaseInfo(r) - return nil - } else if err != nil { - return err - } - } - - chartPath, err := client.ChartPathOptions.LocateChart(args[1], settings) - if err != nil { - return err - } - - v, err := valueOpts.MergeValues(getter.All(settings)) - if err != nil { - return err - } - - // Check chart dependencies to make sure all are present in /charts - ch, err := helmLoader.Load(chartPath) - if err != nil { - return err - } - if req := ch.Metadata.Dependencies; req != nil { - if err := action.CheckDependencies(ch, req); err != nil { - return err - } - } - - if ch.Metadata.Deprecated { - logger.Log.Warningln("This chart is deprecated") - } - - r, err1 := client.Run(args[0], ch, v) - if err1 != nil { - return errors.Wrap(err1, "UPGRADE FAILED") - } - printReleaseInfo(r) - return nil -} - -func runInstall(args []string, client *action.Install, valueOpts *values.Options, settings *cli.EnvSettings) (*release.Release, error) { - if client.Version == "" && client.Devel { - client.Version = ">0.0.0-0" - } - - name, c, err := client.NameAndChart(args) - if err != nil { - return nil, err - } - client.ReleaseName = name - - cp, err := client.ChartPathOptions.LocateChart(c, settings) - if err != nil { - return nil, err - } - - p := getter.All(settings) - vals, err := valueOpts.MergeValues(p) - if err != nil { - return nil, err - } - // Check chart dependencies to make sure all are present in /charts - chartRequested, err := helmLoader.Load(cp) - if err != nil { - return nil, err - } - - if err := checkIfInstallable(chartRequested); err != nil { - return nil, err - } - - if chartRequested.Metadata.Deprecated { - logger.Log.Warningln("This chart is deprecated") - } - - if req := chartRequested.Metadata.Dependencies; req != nil { - if err := action.CheckDependencies(chartRequested, req); err != nil { - if client.DependencyUpdate { - man := &downloader.Manager{ - ChartPath: cp, - Keyring: client.ChartPathOptions.Keyring, - SkipUpdate: false, - Getters: p, - RepositoryConfig: settings.RepositoryConfig, - RepositoryCache: settings.RepositoryCache, - Debug: settings.Debug, - } - if err := man.Update(); err != nil { - return nil, err - } - // Reload the chart with the updated Chart.lock file. - if chartRequested, err = helmLoader.Load(cp); err != nil { - return nil, errors.Wrap(err, "failed reloading chart after repo update") - } - } else { - return nil, err - } - } - } - - return client.Run(chartRequested, vals) -} - -func checkIfInstallable(ch *chart.Chart) error { - switch ch.Metadata.Type { - case "", "application": - return nil - } - return errors.Errorf("%s charts are not installable", ch.Metadata.Type) -} - -func defaultKeyring() string { - if v, ok := os.LookupEnv("GNUPGHOME"); ok { - return filepath.Join(v, "pubring.gpg") - } - return filepath.Join(homedir.HomeDir(), ".gnupg", "pubring.gpg") -} - -func printReleaseInfo(release *release.Release) { - fmt.Printf("NAME: %s\n", release.Name) - if !release.Info.LastDeployed.IsZero() { - fmt.Printf("LAST DEPLOYED: %s\n", release.Info.LastDeployed.Format(time.ANSIC)) - } - fmt.Printf("NAMESPACE: %s\n", release.Namespace) - fmt.Printf("STATUS: %s\n", release.Info.Status.String()) - fmt.Printf("REVISION: %d\n", release.Version) -} diff --git a/cmd/kk/pkg/addons/manifests.go b/cmd/kk/pkg/addons/manifests.go deleted file mode 100644 index 82d16373..00000000 --- a/cmd/kk/pkg/addons/manifests.go +++ /dev/null @@ -1,200 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package addons - -import ( - "errors" - "fmt" - "os" - "path/filepath" - - "k8s.io/apimachinery/pkg/util/sets" - - versionutil "k8s.io/apimachinery/pkg/util/version" - "k8s.io/cli-runtime/pkg/genericclioptions" - "k8s.io/cli-runtime/pkg/printers" - "k8s.io/cli-runtime/pkg/resource" - "k8s.io/client-go/util/homedir" - "k8s.io/kubectl/pkg/cmd/apply" - cmdutil "k8s.io/kubectl/pkg/cmd/util" -) - -var ( - defaultCacheDir = filepath.Join(homedir.HomeDir(), ".kube", "cache") -) - -func InstallYaml(manifests []string, namespace, kubeConfig, version string) error { - - configFlags := NewConfigFlags(kubeConfig, namespace) - o, err := CreateApplyOptions(configFlags, manifests, version) - if err != nil { - return err - } - - if err := o.Run(); err != nil { - return err - } - - return nil -} - -func CreateApplyOptions(configFlags *genericclioptions.ConfigFlags, manifests []string, version string) (*apply.ApplyOptions, error) { - matchVersionKubeConfigFlags := NewMatchVersionFlags(configFlags) - f := cmdutil.NewFactory(matchVersionKubeConfigFlags) - ioStreams := genericclioptions.IOStreams{In: nil, Out: os.Stdout, ErrOut: os.Stderr} - - flags := apply.NewApplyFlags(f, ioStreams) - return ToOptions(flags, manifests, version) -} - -func ToOptions(flags *apply.ApplyFlags, manifests []string, version string) (*apply.ApplyOptions, error) { - serverSideApply := false - cmp, err := versionutil.MustParseSemantic(version).Compare("v1.16.0") - if err != nil { - return nil, errors.New(fmt.Sprintf("Failed to compare version: %v", err)) - } - if cmp == 0 || cmp == 1 { - serverSideApply = true - } else { - serverSideApply = false - } - - dryRunStrategy := cmdutil.DryRunNone - - dynamicClient, err := flags.Factory.DynamicClient() - if err != nil { - return nil, err - } - - dryRunVerifier := resource.NewQueryParamVerifier(dynamicClient, flags.Factory.OpenAPIGetter(), resource.QueryParamDryRun) - fieldValidationVerifier := resource.NewQueryParamVerifier(dynamicClient, flags.Factory.OpenAPIGetter(), resource.QueryParamFieldValidation) - fieldManager := "client-side-apply" - - // allow for a success message operation to be specified at print time - toPrinter := func(operation string) (printers.ResourcePrinter, error) { - flags.PrintFlags.NamePrintFlags.Operation = operation - cmdutil.PrintFlagsWithDryRunStrategy(flags.PrintFlags, dryRunStrategy) - return flags.PrintFlags.ToPrinter() - } - _ = flags.RecordFlags.CompleteWithChangeCause("") - - recorder, err := flags.RecordFlags.ToRecorder() - if err != nil { - return nil, err - } - - filenames := manifests - flags.DeleteFlags.FileNameFlags.Filenames = &filenames - - deleteOptions, err := flags.DeleteFlags.ToOptions(dynamicClient, flags.IOStreams) - if err != nil { - return nil, err - } - - err = deleteOptions.FilenameOptions.RequireFilenameOrKustomize() - if err != nil { - return nil, err - } - - openAPISchema, _ := flags.Factory.OpenAPISchema() - validator, err := flags.Factory.Validator("Ignore", fieldValidationVerifier) - if err != nil { - return nil, err - } - builder := flags.Factory.NewBuilder() - mapper, err := flags.Factory.ToRESTMapper() - if err != nil { - return nil, err - } - - namespace, enforceNamespace, err := flags.Factory.ToRawKubeConfigLoader().Namespace() - if err != nil { - return nil, err - } - - o := &apply.ApplyOptions{ - PrintFlags: flags.PrintFlags, - - DeleteOptions: deleteOptions, - ToPrinter: toPrinter, - ServerSideApply: serverSideApply, - ForceConflicts: true, - FieldManager: fieldManager, - Selector: flags.Selector, - DryRunStrategy: dryRunStrategy, - DryRunVerifier: dryRunVerifier, - Prune: flags.Prune, - PruneResources: flags.PruneResources, - All: flags.All, - Overwrite: flags.Overwrite, - OpenAPIPatch: flags.OpenAPIPatch, - PruneWhitelist: flags.PruneWhitelist, - - Recorder: recorder, - Namespace: namespace, - EnforceNamespace: enforceNamespace, - Validator: validator, - Builder: builder, - Mapper: mapper, - DynamicClient: dynamicClient, - OpenAPISchema: openAPISchema, - - IOStreams: flags.IOStreams, - - VisitedUids: sets.NewString(), - VisitedNamespaces: sets.NewString(), - } - - o.PostProcessorFn = o.PrintAndPrunePostProcessor() - - return o, nil -} - -func NewConfigFlags(kubeconfig, namespace string) *genericclioptions.ConfigFlags { - var impersonateGroup []string - insecure := false - - return &genericclioptions.ConfigFlags{ - Insecure: &insecure, - Timeout: stringptr("0"), - KubeConfig: stringptr(kubeconfig), - - CacheDir: stringptr(defaultCacheDir), - ClusterName: stringptr(""), - AuthInfoName: stringptr(""), - Context: stringptr(""), - Namespace: stringptr(namespace), - APIServer: stringptr(""), - TLSServerName: stringptr(""), - CertFile: stringptr(""), - KeyFile: stringptr(""), - CAFile: stringptr(""), - BearerToken: stringptr(""), - Impersonate: stringptr(""), - ImpersonateGroup: &impersonateGroup, - } -} - -func stringptr(val string) *string { - return &val -} - -func NewMatchVersionFlags(delegate genericclioptions.RESTClientGetter) *cmdutil.MatchVersionFlags { - return &cmdutil.MatchVersionFlags{ - Delegate: delegate, - } -} diff --git a/cmd/kk/pkg/addons/module.go b/cmd/kk/pkg/addons/module.go deleted file mode 100644 index ab833c23..00000000 --- a/cmd/kk/pkg/addons/module.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package addons - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type AddonsModule struct { - common.KubeModule - Skip bool -} - -func (a *AddonsModule) IsSkip() bool { - return a.Skip -} - -func (a *AddonsModule) Init() { - a.Name = "AddonsModule" - a.Desc = "Install addons" - - install := &task.LocalTask{ - Name: "InstallAddons", - Desc: "Install addons", - Action: new(Install), - } - - a.Tasks = []task.Interface{ - install, - } -} diff --git a/cmd/kk/pkg/addons/tasks.go b/cmd/kk/pkg/addons/tasks.go deleted file mode 100644 index cd79d589..00000000 --- a/cmd/kk/pkg/addons/tasks.go +++ /dev/null @@ -1,41 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package addons - -import ( - "fmt" - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -type Install struct { - common.KubeAction -} - -func (i *Install) Execute(runtime connector.Runtime) error { - nums := len(i.KubeConf.Cluster.Addons) - for index, addon := range i.KubeConf.Cluster.Addons { - logger.Log.Messagef(runtime.RemoteHost().GetName(), "Install addon [%v-%v]: %s", nums, index, addon.Name) - if err := InstallAddons(i.KubeConf, &addon, filepath.Join(runtime.GetWorkDir(), fmt.Sprintf("config-%s", runtime.GetObjName()))); err != nil { - return err - } - } - return nil -} diff --git a/cmd/kk/pkg/artifact/manifest.go b/cmd/kk/pkg/artifact/manifest.go deleted file mode 100644 index a3f08003..00000000 --- a/cmd/kk/pkg/artifact/manifest.go +++ /dev/null @@ -1,206 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package artifact - -import ( - "bufio" - "context" - "fmt" - "os" - "sort" - "strings" - - mapset "github.com/deckarep/golang-set" - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - versionutil "k8s.io/apimachinery/pkg/util/version" - - kubekeyv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/client/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -func CreateManifest(arg common.Argument, name string) error { - checkFileExists(arg.FilePath) - - client, err := kubernetes.NewClient(arg.KubeConfig) - if err != nil { - return errors.Wrap(err, "get kubernetes client failed") - } - - nodes, err := client.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) - if err != nil { - return err - } - - archSet := mapset.NewThreadUnsafeSet() - containerSet := mapset.NewThreadUnsafeSet() - imagesSet := mapset.NewThreadUnsafeSet() - osSet := mapset.NewThreadUnsafeSet() - - maxKubeletVersion := versionutil.MustParseGeneric("v0.0.0") - kubernetesDistribution := kubekeyv1alpha2.KubernetesDistribution{} - for _, node := range nodes.Items { - containerStrArr := strings.Split(node.Status.NodeInfo.ContainerRuntimeVersion, "://") - containerRuntime := kubekeyv1alpha2.ContainerRuntime{ - Type: containerStrArr[0], - Version: containerStrArr[1], - } - if containerRuntime.Type == "containerd" && - versionutil.MustParseSemantic(containerRuntime.Version).LessThan(versionutil.MustParseSemantic("1.6.2")) { - containerRuntime.Version = "1.6.2" - } - containerSet.Add(containerRuntime) - - archSet.Add(node.Status.NodeInfo.Architecture) - for _, image := range node.Status.Images { - for _, name := range image.Names { - if !strings.Contains(name, "@sha256") { - if containerRuntime.Type == kubekeyv1alpha2.Docker { - arr := strings.Split(name, "/") - switch len(arr) { - case 1: - name = fmt.Sprintf("docker.io/library/%s", name) - case 2: - name = fmt.Sprintf("docker.io/%s", name) - } - } - imagesSet.Add(name) - } - } - } - - // todo: for now, the cases only have ubuntu, centos. Ant it need to check all linux distribution. - var ( - id, version string - ) - osImageArr := strings.Split(node.Status.NodeInfo.OSImage, " ") - switch strings.ToLower(osImageArr[0]) { - case "ubuntu": - id = "ubuntu" - v := strings.Split(osImageArr[1], ".") - if len(v) >= 2 { - version = fmt.Sprintf("%s.%s", v[0], v[1]) - } - case "centos": - id = "centos" - version = osImageArr[2] - case "rhel": - id = "rhel" - version = osImageArr[2] - default: - id = strings.ToLower(osImageArr[0]) - version = "Can't get the os version. Please edit it manually." - } - - osObj := kubekeyv1alpha2.OperatingSystem{ - Arch: node.Status.NodeInfo.Architecture, - Type: node.Status.NodeInfo.OperatingSystem, - Id: id, - Version: version, - OsImage: node.Status.NodeInfo.OSImage, - } - osSet.Add(osObj) - - kubeletStrArr := strings.Split(node.Status.NodeInfo.KubeletVersion, "+") - kubeletVersion := kubeletStrArr[0] - distribution := "kubernetes" - if len(kubeletStrArr) == 2 { - distribution = "k3s" - } - if maxKubeletVersion.LessThan(versionutil.MustParseGeneric(kubeletVersion)) { - maxKubeletVersion = versionutil.MustParseGeneric(kubeletVersion) - kubernetesDistribution.Version = fmt.Sprintf("v%s", maxKubeletVersion.String()) - kubernetesDistribution.Type = distribution - } - - } - - archArr := make([]string, 0, archSet.Cardinality()) - for _, v := range archSet.ToSlice() { - arch := v.(string) - archArr = append(archArr, arch) - } - imageArr := make([]string, 0, imagesSet.Cardinality()) - for _, v := range imagesSet.ToSlice() { - image := v.(string) - imageArr = append(imageArr, image) - } - osArr := make([]kubekeyv1alpha2.OperatingSystem, 0, osSet.Cardinality()) - for _, v := range osSet.ToSlice() { - osObj := v.(kubekeyv1alpha2.OperatingSystem) - osArr = append(osArr, osObj) - } - containerArr := make([]kubekeyv1alpha2.ContainerRuntime, 0, containerSet.Cardinality()) - for _, v := range containerSet.ToSlice() { - container := v.(kubekeyv1alpha2.ContainerRuntime) - containerArr = append(containerArr, container) - } - - // todo: Whether it need to detect components version - sort.Strings(imageArr) - options := &templates.Options{ - Name: name, - Arches: archArr, - OperatingSystems: osArr, - KubernetesDistributions: []kubekeyv1alpha2.KubernetesDistribution{kubernetesDistribution}, - Components: kubekeyv1alpha2.Components{ - Helm: kubekeyv1alpha2.Helm{Version: kubekeyv1alpha2.DefaultHelmVersion}, - CNI: kubekeyv1alpha2.CNI{Version: kubekeyv1alpha2.DefaultCniVersion}, - ETCD: kubekeyv1alpha2.ETCD{Version: kubekeyv1alpha2.DefaultEtcdVersion}, - Crictl: kubekeyv1alpha2.Crictl{Version: kubekeyv1alpha2.DefaultCrictlVersion}, - Calicoctl: kubekeyv1alpha2.Calicoctl{Version: kubekeyv1alpha2.DefaultCalicoVersion}, - ContainerRuntimes: containerArr, - }, - Images: imageArr, - } - - manifestStr, err := templates.RenderManifest(options) - - if err := os.WriteFile(arg.FilePath, []byte(manifestStr), 0644); err != nil { - return errors.Wrap(err, fmt.Sprintf("write file %s failed", arg.FilePath)) - } - - fmt.Println("Generate KubeKey manifest file successfully") - return nil -} - -func checkFileExists(fileName string) { - if util.IsExist(fileName) { - reader := bufio.NewReader(os.Stdin) - stop := false - for { - if stop { - break - } - fmt.Printf("%s already exists. Are you sure you want to overwrite this file? [yes/no]: ", fileName) - input, _ := reader.ReadString('\n') - input = strings.TrimSpace(input) - - if input != "" { - switch input { - case "yes": - stop = true - case "no": - os.Exit(0) - } - } - } - } -} diff --git a/cmd/kk/pkg/artifact/module.go b/cmd/kk/pkg/artifact/module.go deleted file mode 100644 index d2de0e04..00000000 --- a/cmd/kk/pkg/artifact/module.go +++ /dev/null @@ -1,108 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package artifact - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type RepositoryModule struct { - common.ArtifactModule -} - -func (r *RepositoryModule) Init() { - r.Name = "RepositoryModule" - r.Desc = "Get OS repository ISO file" - - download := &task.LocalTask{ - Name: "DownloadISOFile", - Desc: "Download iso file into work dir", - Prepare: new(EnableDownload), - Action: new(DownloadISOFile), - } - - localCopy := &task.LocalTask{ - Name: "LocalCopy", - Desc: "Copy local iso file into artifact dir", - Action: new(LocalCopy), - } - - r.Tasks = []task.Interface{ - download, - localCopy, - } -} - -type ArchiveModule struct { - common.ArtifactModule -} - -func (a *ArchiveModule) Init() { - a.Name = "ArtifactArchiveModule" - a.Desc = "Archive the dependencies" - - archive := &task.LocalTask{ - Name: "ArchiveDependencies", - Desc: "Archive the dependencies", - Action: new(ArchiveDependencies), - } - - a.Tasks = []task.Interface{ - archive, - } -} - -type UnArchiveModule struct { - common.KubeModule - Skip bool -} - -func (u *UnArchiveModule) IsSkip() bool { - return u.Skip -} - -func (u *UnArchiveModule) Init() { - u.Name = "UnArchiveArtifactModule" - u.Desc = "UnArchive the KubeKey artifact" - - md5Check := &task.LocalTask{ - Name: "CheckArtifactMd5", - Desc: "Check the KubeKey artifact md5 value", - Action: new(Md5Check), - } - - unArchive := &task.LocalTask{ - Name: "UnArchiveArtifact", - Desc: "UnArchive the KubeKey artifact", - Prepare: &Md5AreEqual{Not: true}, - Action: new(UnArchive), - } - - createMd5File := &task.LocalTask{ - Name: "CreateArtifactMd5File", - Desc: "Create the KubeKey artifact Md5 file", - Prepare: &Md5AreEqual{Not: true}, - Action: new(CreateMd5File), - } - - u.Tasks = []task.Interface{ - md5Check, - unArchive, - createMd5File, - } -} diff --git a/cmd/kk/pkg/artifact/prepares.go b/cmd/kk/pkg/artifact/prepares.go deleted file mode 100644 index 3759f11a..00000000 --- a/cmd/kk/pkg/artifact/prepares.go +++ /dev/null @@ -1,54 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package artifact - -import ( - "fmt" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type EnableDownload struct { - common.ArtifactPrepare -} - -func (e *EnableDownload) PreCheck(_ connector.Runtime) (bool, error) { - for _, sys := range e.Manifest.Spec.OperatingSystems { - if sys.Repository.Iso.LocalPath == "" && sys.Repository.Iso.Url != "" { - return true, nil - } - } - return false, nil -} - -type Md5AreEqual struct { - common.KubePrepare - Not bool -} - -func (m *Md5AreEqual) PreCheck(_ connector.Runtime) (bool, error) { - equal, ok := m.ModuleCache.GetMustBool("md5AreEqual") - if !ok { - return false, fmt.Errorf("get md5 equal value from module cache failed") - } - - if equal { - return !m.Not, nil - } - return m.Not, nil -} diff --git a/cmd/kk/pkg/artifact/tasks.go b/cmd/kk/pkg/artifact/tasks.go deleted file mode 100644 index 89fb033b..00000000 --- a/cmd/kk/pkg/artifact/tasks.go +++ /dev/null @@ -1,172 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package artifact - -import ( - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - coreutil "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type DownloadISOFile struct { - common.ArtifactAction -} - -func (d *DownloadISOFile) Execute(runtime connector.Runtime) error { - for i, sys := range d.Manifest.Spec.OperatingSystems { - if sys.Repository.Iso.Url == "" { - continue - } - - fileName := fmt.Sprintf("%s-%s-%s.iso", sys.Id, sys.Version, sys.Arch) - filePath := filepath.Join(runtime.GetWorkDir(), fileName) - getCmd := d.Manifest.Arg.DownloadCommand(filePath, sys.Repository.Iso.Url) - - cmd := exec.Command("/bin/sh", "-c", getCmd) - stdout, err := cmd.StdoutPipe() - if err != nil { - return fmt.Errorf("Failed to download %s iso file: %s error: %w ", fileName, getCmd, err) - } - cmd.Stderr = cmd.Stdout - - if err = cmd.Start(); err != nil { - return fmt.Errorf("Failed to download %s iso file: %s error: %w ", fileName, getCmd, err) - } - for { - tmp := make([]byte, 1024) - _, err := stdout.Read(tmp) - fmt.Print(string(tmp)) - if errors.Is(err, io.EOF) { - break - } else if err != nil { - logger.Log.Errorln(err) - break - } - } - if err = cmd.Wait(); err != nil { - return fmt.Errorf("Failed to download %s iso file: %s error: %w ", fileName, getCmd, err) - } - d.Manifest.Spec.OperatingSystems[i].Repository.Iso.LocalPath = filePath - } - return nil -} - -type LocalCopy struct { - common.ArtifactAction -} - -func (l *LocalCopy) Execute(runtime connector.Runtime) error { - for _, sys := range l.Manifest.Spec.OperatingSystems { - if sys.Repository.Iso.LocalPath == "" { - continue - } - - dir := filepath.Join(runtime.GetWorkDir(), common.Artifact, "repository", sys.Arch, sys.Id, sys.Version) - if err := coreutil.Mkdir(dir); err != nil { - return errors.Wrapf(errors.WithStack(err), "mkdir %s failed", dir) - } - - path := filepath.Join(dir, fmt.Sprintf("%s-%s-%s.iso", sys.Id, sys.Version, sys.Arch)) - if out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("sudo cp -f %s %s", sys.Repository.Iso.LocalPath, path)).CombinedOutput(); err != nil { - return errors.Errorf("copy %s to %s failed: %s", sys.Repository.Iso.LocalPath, path, string(out)) - } - } - - return nil -} - -type ArchiveDependencies struct { - common.ArtifactAction -} - -func (a *ArchiveDependencies) Execute(runtime connector.Runtime) error { - src := filepath.Join(runtime.GetWorkDir(), common.Artifact) - if err := coreutil.Tar(src, a.Manifest.Arg.Output, src); err != nil { - return errors.Wrapf(errors.WithStack(err), "archive %s failed", src) - } - - // remove the src directory - if err := os.RemoveAll(src); err != nil { - return errors.Wrapf(errors.WithStack(err), "remove %s failed", src) - } - return nil -} - -type UnArchive struct { - common.KubeAction -} - -func (u *UnArchive) Execute(runtime connector.Runtime) error { - if err := coreutil.Untar(u.KubeConf.Arg.Artifact, runtime.GetWorkDir()); err != nil { - return errors.Wrapf(errors.WithStack(err), "unArchive %s failed", u.KubeConf.Arg.Artifact) - } - return nil -} - -type Md5Check struct { - common.KubeAction -} - -func (m *Md5Check) Execute(runtime connector.Runtime) error { - m.ModuleCache.Set("md5AreEqual", false) - - // check if there is a md5.sum file. This file's content contains the last artifact md5 value. - oldFile := filepath.Join(runtime.GetWorkDir(), "artifact.md5") - if exist := coreutil.IsExist(oldFile); !exist { - return nil - } - - oldMd5, err := os.ReadFile(oldFile) - if err != nil { - return errors.Wrapf(errors.WithStack(err), "read old md5 file %s failed", oldFile) - } - - newMd5 := coreutil.LocalMd5Sum(m.KubeConf.Arg.Artifact) - - if string(oldMd5) == newMd5 { - m.ModuleCache.Set("md5AreEqual", true) - } - return nil -} - -type CreateMd5File struct { - common.KubeAction -} - -func (c *CreateMd5File) Execute(runtime connector.Runtime) error { - oldFile := filepath.Join(runtime.GetWorkDir(), "artifact.md5") - newMd5 := coreutil.LocalMd5Sum(c.KubeConf.Arg.Artifact) - f, err := os.Create(oldFile) - if err != nil { - return errors.Wrapf(errors.WithStack(err), "create md5 fild %s failed", oldFile) - } - - if _, err := io.Copy(f, strings.NewReader(newMd5)); err != nil { - return errors.Wrapf(errors.WithStack(err), "write md5 value to file %s failed", oldFile) - } - return nil -} diff --git a/cmd/kk/pkg/artifact/templates/manifest.go b/cmd/kk/pkg/artifact/templates/manifest.go deleted file mode 100644 index 7b88d4fa..00000000 --- a/cmd/kk/pkg/artifact/templates/manifest.go +++ /dev/null @@ -1,100 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" - - kubekeyv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -// Manifest defines the template of manifest file. -var Manifest = template.Must(template.New("Spec").Parse( - dedent.Dedent(` -apiVersion: kubekey.kubesphere.io/v1alpha2 -kind: Manifest -metadata: - name: {{ .Options.Name }} -spec: - arches: - {{- range .Options.Arches }} - - {{ . }} - {{- end }} - operatingSystems: - {{- range $i, $v := .Options.OperatingSystems }} - - arch: {{ $v.Arch }} - type: {{ $v.Type }} - id: {{ $v.Id }} - version: "{{ $v.Version }}" - osImage: {{ $v.OsImage }} - repository: - iso: - localPath: - url: - {{- end }} - kubernetesDistributions: - {{- range $i, $v := .Options.KubernetesDistributions }} - - type: {{ $v.Type }} - version: {{ $v.Version }} - {{- end}} - components: - helm: - version: {{ .Options.Components.Helm.Version }} - cni: - version: {{ .Options.Components.CNI.Version }} - etcd: - version: {{ .Options.Components.ETCD.Version }} - containerRuntimes: - {{- range $i, $v := .Options.Components.ContainerRuntimes }} - - type: {{ $v.Type }} - version: {{ $v.Version }} - {{- end}} - crictl: - version: {{ .Options.Components.Crictl.Version }} - ## - # docker-registry: - # version: "2" - # harbor: - # version: v2.4.1 - # docker-compose: - # version: v2.2.2 - images: - {{- range .Options.Images }} - - {{ . }} - {{- end }} - registry: - auths: {} - - `))) - -type Options struct { - Name string - Arches []string - OperatingSystems []kubekeyv1alpha2.OperatingSystem - KubernetesDistributions []kubekeyv1alpha2.KubernetesDistribution - Components kubekeyv1alpha2.Components - Images []string -} - -func RenderManifest(opt *Options) (string, error) { - return util.Render(Manifest, util.Data{ - "Options": opt, - }) -} diff --git a/cmd/kk/pkg/binaries/k3s.go b/cmd/kk/pkg/binaries/k3s.go deleted file mode 100644 index 0b6e9325..00000000 --- a/cmd/kk/pkg/binaries/k3s.go +++ /dev/null @@ -1,129 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binaries - -import ( - "fmt" - "os/exec" - - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" -) - -// K3sFilesDownloadHTTP defines the kubernetes' binaries that need to be downloaded in advance and downloads them. -func K3sFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string, pipelineCache *cache.Cache) error { - - etcd := files.NewKubeBinary("etcd", arch, kubekeyapiv1alpha2.DefaultEtcdVersion, path, kubeConf.Arg.DownloadCommand) - kubecni := files.NewKubeBinary("kubecni", arch, kubekeyapiv1alpha2.DefaultCniVersion, path, kubeConf.Arg.DownloadCommand) - helm := files.NewKubeBinary("helm", arch, kubekeyapiv1alpha2.DefaultHelmVersion, path, kubeConf.Arg.DownloadCommand) - k3s := files.NewKubeBinary("k3s", arch, version, path, kubeConf.Arg.DownloadCommand) - calicoctl := files.NewKubeBinary("calicoctl", arch, kubekeyapiv1alpha2.DefaultCalicoVersion, path, kubeConf.Arg.DownloadCommand) - - binaries := []*files.KubeBinary{k3s, helm, kubecni, etcd} - - if kubeConf.Cluster.Network.Plugin == "calico" { - binaries = append(binaries, calicoctl) - } - - binariesMap := make(map[string]*files.KubeBinary) - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - - binariesMap[binary.ID] = binary - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - p := binary.Path() - if err := binary.SHA256Check(); err != nil { - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", p)).Run() - } else { - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - - pipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap) - return nil -} - -func K3sArtifactBinariesDownload(manifest *common.ArtifactManifest, path, arch, version string) error { - m := manifest.Spec - - etcd := files.NewKubeBinary("etcd", arch, m.Components.ETCD.Version, path, manifest.Arg.DownloadCommand) - kubecni := files.NewKubeBinary("kubecni", arch, m.Components.CNI.Version, path, manifest.Arg.DownloadCommand) - helm := files.NewKubeBinary("helm", arch, m.Components.Helm.Version, path, manifest.Arg.DownloadCommand) - k3s := files.NewKubeBinary("k3s", arch, version, path, manifest.Arg.DownloadCommand) - crictl := files.NewKubeBinary("crictl", arch, m.Components.Crictl.Version, path, manifest.Arg.DownloadCommand) - binaries := []*files.KubeBinary{k3s, helm, kubecni, etcd} - - dockerArr := make([]*files.KubeBinary, 0, 0) - dockerVersionMap := make(map[string]struct{}) - for _, c := range m.Components.ContainerRuntimes { - var dockerVersion string - if c.Type == common.Docker { - dockerVersion = c.Version - } else { - dockerVersion = kubekeyapiv1alpha2.DefaultDockerVersion - } - if _, ok := dockerVersionMap[dockerVersion]; !ok { - dockerVersionMap[dockerVersion] = struct{}{} - docker := files.NewKubeBinary("docker", arch, dockerVersion, path, manifest.Arg.DownloadCommand) - dockerArr = append(dockerArr, docker) - } - } - - binaries = append(binaries, dockerArr...) - if m.Components.Crictl.Version != "" { - binaries = append(binaries, crictl) - } - - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - if err := binary.SHA256Check(); err != nil { - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", binary.Path())).Run() - } else { - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - - return nil -} diff --git a/cmd/kk/pkg/binaries/k8e.go b/cmd/kk/pkg/binaries/k8e.go deleted file mode 100644 index 05b2eb61..00000000 --- a/cmd/kk/pkg/binaries/k8e.go +++ /dev/null @@ -1,123 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binaries - -import ( - "fmt" - "os/exec" - - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" -) - -// K8eFilesDownloadHTTP defines the kubernetes' binaries that need to be downloaded in advance and downloads them. -func K8eFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string, pipelineCache *cache.Cache) error { - - etcd := files.NewKubeBinary("etcd", arch, kubekeyapiv1alpha2.DefaultEtcdVersion, path, kubeConf.Arg.DownloadCommand) - kubecni := files.NewKubeBinary("kubecni", arch, kubekeyapiv1alpha2.DefaultCniVersion, path, kubeConf.Arg.DownloadCommand) - helm := files.NewKubeBinary("helm", arch, kubekeyapiv1alpha2.DefaultHelmVersion, path, kubeConf.Arg.DownloadCommand) - k8e := files.NewKubeBinary("k8e", arch, version, path, kubeConf.Arg.DownloadCommand) - - binaries := []*files.KubeBinary{k8e, helm, kubecni, etcd} - binariesMap := make(map[string]*files.KubeBinary) - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - - binariesMap[binary.ID] = binary - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - p := binary.Path() - if err := binary.SHA256Check(); err != nil { - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", p)).Run() - } else { - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - - pipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap) - return nil -} - -func K8eArtifactBinariesDownload(manifest *common.ArtifactManifest, path, arch, version string) error { - m := manifest.Spec - - etcd := files.NewKubeBinary("etcd", arch, m.Components.ETCD.Version, path, manifest.Arg.DownloadCommand) - kubecni := files.NewKubeBinary("kubecni", arch, m.Components.CNI.Version, path, manifest.Arg.DownloadCommand) - helm := files.NewKubeBinary("helm", arch, m.Components.Helm.Version, path, manifest.Arg.DownloadCommand) - k8e := files.NewKubeBinary("k8e", arch, version, path, manifest.Arg.DownloadCommand) - crictl := files.NewKubeBinary("crictl", arch, m.Components.Crictl.Version, path, manifest.Arg.DownloadCommand) - binaries := []*files.KubeBinary{k8e, helm, kubecni, etcd} - - dockerArr := make([]*files.KubeBinary, 0, 0) - dockerVersionMap := make(map[string]struct{}) - for _, c := range m.Components.ContainerRuntimes { - var dockerVersion string - if c.Type == common.Docker { - dockerVersion = c.Version - } else { - dockerVersion = kubekeyapiv1alpha2.DefaultDockerVersion - } - if _, ok := dockerVersionMap[dockerVersion]; !ok { - dockerVersionMap[dockerVersion] = struct{}{} - docker := files.NewKubeBinary("docker", arch, dockerVersion, path, manifest.Arg.DownloadCommand) - dockerArr = append(dockerArr, docker) - } - } - - binaries = append(binaries, dockerArr...) - if m.Components.Crictl.Version != "" { - binaries = append(binaries, crictl) - } - - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - if err := binary.SHA256Check(); err != nil { - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", binary.Path())).Run() - } else { - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - - return nil -} diff --git a/cmd/kk/pkg/binaries/kubernetes.go b/cmd/kk/pkg/binaries/kubernetes.go deleted file mode 100644 index 4e381305..00000000 --- a/cmd/kk/pkg/binaries/kubernetes.go +++ /dev/null @@ -1,199 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binaries - -import ( - "fmt" - "os/exec" - - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" -) - -// K8sFilesDownloadHTTP defines the kubernetes' binaries that need to be downloaded in advance and downloads them. -func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string, pipelineCache *cache.Cache) error { - - etcd := files.NewKubeBinary("etcd", arch, kubekeyapiv1alpha2.DefaultEtcdVersion, path, kubeConf.Arg.DownloadCommand) - kubeadm := files.NewKubeBinary("kubeadm", arch, version, path, kubeConf.Arg.DownloadCommand) - kubelet := files.NewKubeBinary("kubelet", arch, version, path, kubeConf.Arg.DownloadCommand) - kubectl := files.NewKubeBinary("kubectl", arch, version, path, kubeConf.Arg.DownloadCommand) - kubecni := files.NewKubeBinary("kubecni", arch, kubekeyapiv1alpha2.DefaultCniVersion, path, kubeConf.Arg.DownloadCommand) - helm := files.NewKubeBinary("helm", arch, kubekeyapiv1alpha2.DefaultHelmVersion, path, kubeConf.Arg.DownloadCommand) - docker := files.NewKubeBinary("docker", arch, kubekeyapiv1alpha2.DefaultDockerVersion, path, kubeConf.Arg.DownloadCommand) - crictl := files.NewKubeBinary("crictl", arch, kubekeyapiv1alpha2.DefaultCrictlVersion, path, kubeConf.Arg.DownloadCommand) - containerd := files.NewKubeBinary("containerd", arch, kubekeyapiv1alpha2.DefaultContainerdVersion, path, kubeConf.Arg.DownloadCommand) - runc := files.NewKubeBinary("runc", arch, kubekeyapiv1alpha2.DefaultRuncVersion, path, kubeConf.Arg.DownloadCommand) - calicoctl := files.NewKubeBinary("calicoctl", arch, kubekeyapiv1alpha2.DefaultCalicoVersion, path, kubeConf.Arg.DownloadCommand) - - binaries := []*files.KubeBinary{kubeadm, kubelet, kubectl, helm, kubecni, crictl, etcd} - - if kubeConf.Cluster.Kubernetes.ContainerManager == kubekeyapiv1alpha2.Docker { - binaries = append(binaries, docker) - } else if kubeConf.Cluster.Kubernetes.ContainerManager == kubekeyapiv1alpha2.Containerd { - binaries = append(binaries, containerd, runc) - } - - if kubeConf.Cluster.Network.Plugin == "calico" { - binaries = append(binaries, calicoctl) - } - - binariesMap := make(map[string]*files.KubeBinary) - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - - binariesMap[binary.ID] = binary - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - if err := binary.SHA256Check(); err != nil { - p := binary.Path() - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", p)).Run() - } else { - logger.Log.Messagef(common.LocalHost, "%s exists", binary.ID) - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - - if kubeConf.Cluster.KubeSphere.Version == "v2.1.1" { - logger.Log.Infoln(fmt.Sprintf("Downloading %s ...", "helm2")) - if util.IsExist(fmt.Sprintf("%s/helm2", helm.BaseDir)) == false { - cmd := kubeConf.Arg.DownloadCommand(fmt.Sprintf("%s/helm2", helm.BaseDir), - fmt.Sprintf("https://kubernetes-helm.pek3b.qingstor.com/linux-%s/%s/helm", helm.Arch, "v2.16.9")) - if output, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput(); err != nil { - fmt.Println(string(output)) - return errors.Wrap(err, "Failed to download helm2 binary") - } - } - } - - pipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap) - return nil -} - -func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path, arch, k8sVersion string) error { - m := manifest.Spec - - etcd := files.NewKubeBinary("etcd", arch, m.Components.ETCD.Version, path, manifest.Arg.DownloadCommand) - kubeadm := files.NewKubeBinary("kubeadm", arch, k8sVersion, path, manifest.Arg.DownloadCommand) - kubelet := files.NewKubeBinary("kubelet", arch, k8sVersion, path, manifest.Arg.DownloadCommand) - kubectl := files.NewKubeBinary("kubectl", arch, k8sVersion, path, manifest.Arg.DownloadCommand) - kubecni := files.NewKubeBinary("kubecni", arch, m.Components.CNI.Version, path, manifest.Arg.DownloadCommand) - helm := files.NewKubeBinary("helm", arch, m.Components.Helm.Version, path, manifest.Arg.DownloadCommand) - crictl := files.NewKubeBinary("crictl", arch, m.Components.Crictl.Version, path, manifest.Arg.DownloadCommand) - calicoctl := files.NewKubeBinary("calicoctl", arch, m.Components.Calicoctl.Version, path, manifest.Arg.DownloadCommand) - binaries := []*files.KubeBinary{kubeadm, kubelet, kubectl, helm, kubecni, etcd, calicoctl} - - containerManagerArr := make([]*files.KubeBinary, 0, 0) - containerManagerVersion := make(map[string]struct{}) - for _, c := range m.Components.ContainerRuntimes { - if _, ok := containerManagerVersion[c.Type+c.Version]; !ok { - containerManagerVersion[c.Type+c.Version] = struct{}{} - containerManager := files.NewKubeBinary(c.Type, arch, c.Version, path, manifest.Arg.DownloadCommand) - containerManagerArr = append(containerManagerArr, containerManager) - if c.Type == "containerd" { - runc := files.NewKubeBinary("runc", arch, kubekeyapiv1alpha2.DefaultRuncVersion, path, manifest.Arg.DownloadCommand) - containerManagerArr = append(containerManagerArr, runc) - } - } - } - - binaries = append(binaries, containerManagerArr...) - if m.Components.Crictl.Version != "" { - binaries = append(binaries, crictl) - } - - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - if err := binary.SHA256Check(); err != nil { - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", binary.Path())).Run() - } else { - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - - return nil -} - -// CriDownloadHTTP defines the kubernetes' binaries that need to be downloaded in advance and downloads them. -func CriDownloadHTTP(kubeConf *common.KubeConf, path, arch string, pipelineCache *cache.Cache) error { - - binaries := []*files.KubeBinary{} - switch kubeConf.Arg.Type { - case common.Docker: - docker := files.NewKubeBinary("docker", arch, kubekeyapiv1alpha2.DefaultDockerVersion, path, kubeConf.Arg.DownloadCommand) - binaries = append(binaries, docker) - case common.Containerd: - containerd := files.NewKubeBinary("containerd", arch, kubekeyapiv1alpha2.DefaultContainerdVersion, path, kubeConf.Arg.DownloadCommand) - runc := files.NewKubeBinary("runc", arch, kubekeyapiv1alpha2.DefaultRuncVersion, path, kubeConf.Arg.DownloadCommand) - crictl := files.NewKubeBinary("crictl", arch, kubekeyapiv1alpha2.DefaultCrictlVersion, path, kubeConf.Arg.DownloadCommand) - binaries = append(binaries, containerd, runc, crictl) - default: - } - binariesMap := make(map[string]*files.KubeBinary) - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - - binariesMap[binary.ID] = binary - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - if err := binary.SHA256Check(); err != nil { - p := binary.Path() - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", p)).Run() - } else { - logger.Log.Messagef(common.LocalHost, "%s exists", binary.ID) - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - - pipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap) - return nil -} diff --git a/cmd/kk/pkg/binaries/module.go b/cmd/kk/pkg/binaries/module.go deleted file mode 100644 index 6facc57f..00000000 --- a/cmd/kk/pkg/binaries/module.go +++ /dev/null @@ -1,193 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binaries - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type NodeBinariesModule struct { - common.KubeModule -} - -func (n *NodeBinariesModule) Init() { - n.Name = "NodeBinariesModule" - n.Desc = "Download installation binaries" - - download := &task.LocalTask{ - Name: "DownloadBinaries", - Desc: "Download installation binaries", - Action: new(Download), - } - - n.Tasks = []task.Interface{ - download, - } -} - -type K3sNodeBinariesModule struct { - common.KubeModule -} - -func (k *K3sNodeBinariesModule) Init() { - k.Name = "K3sNodeBinariesModule" - k.Desc = "Download installation binaries" - - download := &task.LocalTask{ - Name: "DownloadBinaries", - Desc: "Download installation binaries", - Action: new(K3sDownload), - } - - k.Tasks = []task.Interface{ - download, - } -} - -type K8eNodeBinariesModule struct { - common.KubeModule -} - -func (k *K8eNodeBinariesModule) Init() { - k.Name = "K8eNodeBinariesModule" - k.Desc = "Download installation binaries" - - download := &task.LocalTask{ - Name: "DownloadBinaries", - Desc: "Download installation binaries", - Action: new(K8eDownload), - } - - k.Tasks = []task.Interface{ - download, - } -} - -type ArtifactBinariesModule struct { - common.ArtifactModule -} - -func (a *ArtifactBinariesModule) Init() { - a.Name = "ArtifactBinariesModule" - a.Desc = "Download artifact binaries" - - download := &task.LocalTask{ - Name: "DownloadBinaries", - Desc: "Download manifest expect binaries", - Action: new(ArtifactDownload), - } - - a.Tasks = []task.Interface{ - download, - } -} - -type K3sArtifactBinariesModule struct { - common.ArtifactModule -} - -func (a *K3sArtifactBinariesModule) Init() { - a.Name = "K3sArtifactBinariesModule" - a.Desc = "Download artifact binaries" - - download := &task.LocalTask{ - Name: "K3sDownloadBinaries", - Desc: "Download k3s manifest expect binaries", - Action: new(K3sArtifactDownload), - } - - a.Tasks = []task.Interface{ - download, - } -} - -type K8eArtifactBinariesModule struct { - common.ArtifactModule -} - -func (a *K8eArtifactBinariesModule) Init() { - a.Name = "K8eArtifactBinariesModule" - a.Desc = "Download artifact binaries" - - download := &task.LocalTask{ - Name: "K8eDownloadBinaries", - Desc: "Download k8e manifest expect binaries", - Action: new(K8eArtifactDownload), - } - - a.Tasks = []task.Interface{ - download, - } -} - -type RegistryPackageModule struct { - common.KubeModule -} - -func (n *RegistryPackageModule) Init() { - n.Name = "RegistryPackageModule" - n.Desc = "Download registry package" - - if len(n.Runtime.GetHostsByRole(common.Registry)) == 0 { - logger.Log.Fatal(errors.New("[registry] node not found in the roleGroups of the configuration file")) - } - - download := &task.LocalTask{ - Name: "DownloadRegistryPackage", - Desc: "Download registry package", - Action: new(RegistryPackageDownload), - } - - n.Tasks = []task.Interface{ - download, - } -} - -type CriBinariesModule struct { - common.KubeModule -} - -func (i *CriBinariesModule) Init() { - i.Name = "CriBinariesModule" - i.Desc = "Download Cri package" - switch i.KubeConf.Arg.Type { - case common.Docker: - i.Tasks = CriBinaries(i) - case common.Containerd: - i.Tasks = CriBinaries(i) - default: - } - -} - -func CriBinaries(p *CriBinariesModule) []task.Interface { - - download := &task.LocalTask{ - Name: "DownloadCriPackage", - Desc: "Download Cri package", - Action: new(CriDownload), - } - - p.Tasks = []task.Interface{ - download, - } - return p.Tasks -} diff --git a/cmd/kk/pkg/binaries/registry.go b/cmd/kk/pkg/binaries/registry.go deleted file mode 100644 index ec31028a..00000000 --- a/cmd/kk/pkg/binaries/registry.go +++ /dev/null @@ -1,126 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binaries - -import ( - "fmt" - "os/exec" - - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" -) - -// RegistryPackageDownloadHTTP defines the kubernetes' binaries that need to be downloaded in advance and downloads them. -func RegistryPackageDownloadHTTP(kubeConf *common.KubeConf, path, arch string, pipelineCache *cache.Cache) error { - var binaries []*files.KubeBinary - - switch kubeConf.Cluster.Registry.Type { - case common.Harbor: - // TODO: Harbor only supports amd64, so there is no need to consider other architectures at present. - harbor := files.NewKubeBinary("harbor", arch, kubekeyapiv1alpha2.DefaultHarborVersion, path, kubeConf.Arg.DownloadCommand) - compose := files.NewKubeBinary("compose", arch, kubekeyapiv1alpha2.DefaultDockerComposeVersion, path, kubeConf.Arg.DownloadCommand) - docker := files.NewKubeBinary("docker", arch, kubekeyapiv1alpha2.DefaultDockerVersion, path, kubeConf.Arg.DownloadCommand) - binaries = []*files.KubeBinary{harbor, docker, compose} - default: - registry := files.NewKubeBinary("registry", arch, kubekeyapiv1alpha2.DefaultRegistryVersion, path, kubeConf.Arg.DownloadCommand) - binaries = []*files.KubeBinary{registry} - } - - binariesMap := make(map[string]*files.KubeBinary) - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - binariesMap[binary.ID] = binary - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - if err := binary.SHA256Check(); err != nil { - p := binary.Path() - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", p)).Run() - } else { - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - - pipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap) - return nil -} - -func RegistryBinariesDownload(manifest *common.ArtifactManifest, path, arch string) error { - - m := manifest.Spec - binaries := make([]*files.KubeBinary, 0, 0) - - if m.Components.DockerRegistry.Version != "" { - registry := files.NewKubeBinary("registry", arch, kubekeyapiv1alpha2.DefaultRegistryVersion, path, manifest.Arg.DownloadCommand) - binaries = append(binaries, registry) - } - - if m.Components.Harbor.Version != "" { - harbor := files.NewKubeBinary("harbor", arch, kubekeyapiv1alpha2.DefaultHarborVersion, path, manifest.Arg.DownloadCommand) - if arch == "amd64" { - binaries = append(binaries, harbor) - } else { - logger.Log.Warningf("Harbor only supports amd64, the KubeKey artifact will not contain the Harbor %s", arch) - } - } - - if m.Components.DockerCompose.Version != "" { - compose := files.NewKubeBinary("compose", arch, kubekeyapiv1alpha2.DefaultDockerComposeVersion, path, manifest.Arg.DownloadCommand) - containerManager := files.NewKubeBinary("docker", arch, kubekeyapiv1alpha2.DefaultDockerVersion, path, manifest.Arg.DownloadCommand) - if arch == "amd64" { - binaries = append(binaries, compose) - binaries = append(binaries, containerManager) - } - } - - for _, binary := range binaries { - if err := binary.CreateBaseDir(); err != nil { - return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) - } - - logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) - - if util.IsExist(binary.Path()) { - // download it again if it's incorrect - if err := binary.SHA256Check(); err != nil { - p := binary.Path() - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", p)).Run() - } else { - continue - } - } - - if err := binary.Download(); err != nil { - return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) - } - } - return nil -} diff --git a/cmd/kk/pkg/binaries/tasks.go b/cmd/kk/pkg/binaries/tasks.go deleted file mode 100644 index 7d829cb7..00000000 --- a/cmd/kk/pkg/binaries/tasks.go +++ /dev/null @@ -1,305 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binaries - -import ( - "fmt" - "path/filepath" - - mapset "github.com/deckarep/golang-set" - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type Download struct { - common.KubeAction -} - -func (d *Download) Execute(runtime connector.Runtime) error { - cfg := d.KubeConf.Cluster - - var kubeVersion string - if cfg.Kubernetes.Version == "" { - kubeVersion = kubekeyapiv1alpha2.DefaultKubeVersion - } else { - kubeVersion = cfg.Kubernetes.Version - } - - archMap := make(map[string]bool) - for _, host := range cfg.Hosts { - switch host.Arch { - case "amd64": - archMap["amd64"] = true - case "arm64": - archMap["arm64"] = true - default: - return errors.New(fmt.Sprintf("Unsupported architecture: %s", host.Arch)) - } - } - - for arch := range archMap { - if err := K8sFilesDownloadHTTP(d.KubeConf, runtime.GetWorkDir(), kubeVersion, arch, d.PipelineCache); err != nil { - return err - } - } - return nil -} - -type K3sDownload struct { - common.KubeAction -} - -func (k *K3sDownload) Execute(runtime connector.Runtime) error { - cfg := k.KubeConf.Cluster - - var kubeVersion string - if cfg.Kubernetes.Version == "" { - kubeVersion = kubekeyapiv1alpha2.DefaultKubeVersion - } else { - kubeVersion = cfg.Kubernetes.Version - } - - archMap := make(map[string]bool) - for _, host := range cfg.Hosts { - switch host.Arch { - case "amd64": - archMap["amd64"] = true - case "arm64": - archMap["arm64"] = true - default: - return errors.New(fmt.Sprintf("Unsupported architecture: %s", host.Arch)) - } - } - - for arch := range archMap { - if err := K3sFilesDownloadHTTP(k.KubeConf, runtime.GetWorkDir(), kubeVersion, arch, k.PipelineCache); err != nil { - return err - } - } - return nil -} - -type K8eDownload struct { - common.KubeAction -} - -func (k *K8eDownload) Execute(runtime connector.Runtime) error { - cfg := k.KubeConf.Cluster - - var kubeVersion string - if cfg.Kubernetes.Version == "" { - kubeVersion = kubekeyapiv1alpha2.DefaultKubeVersion - } else { - kubeVersion = cfg.Kubernetes.Version - } - - archMap := make(map[string]bool) - for _, host := range cfg.Hosts { - switch host.Arch { - case "amd64": - archMap["amd64"] = true - case "arm64": - archMap["arm64"] = true - default: - return errors.New(fmt.Sprintf("Unsupported architecture: %s", host.Arch)) - } - } - - for arch := range archMap { - if err := K8eFilesDownloadHTTP(k.KubeConf, runtime.GetWorkDir(), kubeVersion, arch, k.PipelineCache); err != nil { - return err - } - } - return nil -} - -type ArtifactDownload struct { - common.ArtifactAction -} - -func (a *ArtifactDownload) Execute(runtime connector.Runtime) error { - manifest := a.Manifest.Spec - - archMap := make(map[string]bool) - for _, arch := range manifest.Arches { - switch arch { - case "amd64": - archMap["amd64"] = true - case "arm64": - archMap["arm64"] = true - default: - return errors.New(fmt.Sprintf("Unsupported architecture: %s", arch)) - } - } - - kubernetesSet := mapset.NewThreadUnsafeSet() - for _, k := range manifest.KubernetesDistributions { - kubernetesSet.Add(k) - } - - kubernetesVersions := make([]string, 0, kubernetesSet.Cardinality()) - for _, k := range kubernetesSet.ToSlice() { - k8s := k.(kubekeyapiv1alpha2.KubernetesDistribution) - kubernetesVersions = append(kubernetesVersions, k8s.Version) - } - - basePath := filepath.Join(runtime.GetWorkDir(), common.Artifact) - for arch := range archMap { - for _, version := range kubernetesVersions { - if err := KubernetesArtifactBinariesDownload(a.Manifest, basePath, arch, version); err != nil { - return err - } - } - - if err := RegistryBinariesDownload(a.Manifest, basePath, arch); err != nil { - return err - } - } - return nil -} - -type K3sArtifactDownload struct { - common.ArtifactAction -} - -func (a *K3sArtifactDownload) Execute(runtime connector.Runtime) error { - manifest := a.Manifest.Spec - - archMap := make(map[string]bool) - for _, arch := range manifest.Arches { - switch arch { - case "amd64": - archMap["amd64"] = true - case "arm64": - archMap["arm64"] = true - default: - return errors.New(fmt.Sprintf("Unsupported architecture: %s", arch)) - } - } - - kubernetesSet := mapset.NewThreadUnsafeSet() - for _, k := range manifest.KubernetesDistributions { - kubernetesSet.Add(k) - } - - kubernetesVersions := make([]string, 0, kubernetesSet.Cardinality()) - for _, k := range kubernetesSet.ToSlice() { - k8s := k.(kubekeyapiv1alpha2.KubernetesDistribution) - kubernetesVersions = append(kubernetesVersions, k8s.Version) - } - - basePath := filepath.Join(runtime.GetWorkDir(), common.Artifact) - for arch := range archMap { - for _, version := range kubernetesVersions { - if err := K3sArtifactBinariesDownload(a.Manifest, basePath, arch, version); err != nil { - return err - } - } - - if err := RegistryBinariesDownload(a.Manifest, basePath, arch); err != nil { - return err - } - } - return nil -} - -type K8eArtifactDownload struct { - common.ArtifactAction -} - -func (a *K8eArtifactDownload) Execute(runtime connector.Runtime) error { - manifest := a.Manifest.Spec - - archMap := make(map[string]bool) - for _, arch := range manifest.Arches { - switch arch { - case "amd64": - archMap["amd64"] = true - case "arm64": - archMap["arm64"] = true - default: - return errors.New(fmt.Sprintf("Unsupported architecture: %s", arch)) - } - } - - kubernetesSet := mapset.NewThreadUnsafeSet() - for _, k := range manifest.KubernetesDistributions { - kubernetesSet.Add(k) - } - - kubernetesVersions := make([]string, 0, kubernetesSet.Cardinality()) - for _, k := range kubernetesSet.ToSlice() { - k8s := k.(kubekeyapiv1alpha2.KubernetesDistribution) - kubernetesVersions = append(kubernetesVersions, k8s.Version) - } - - basePath := filepath.Join(runtime.GetWorkDir(), common.Artifact) - for arch := range archMap { - for _, version := range kubernetesVersions { - if err := K8eArtifactBinariesDownload(a.Manifest, basePath, arch, version); err != nil { - return err - } - } - - if err := RegistryBinariesDownload(a.Manifest, basePath, arch); err != nil { - return err - } - } - return nil -} - -type RegistryPackageDownload struct { - common.KubeAction -} - -func (k *RegistryPackageDownload) Execute(runtime connector.Runtime) error { - arch := runtime.GetHostsByRole(common.Registry)[0].GetArch() - if err := RegistryPackageDownloadHTTP(k.KubeConf, runtime.GetWorkDir(), arch, k.PipelineCache); err != nil { - return err - } - - return nil -} - -type CriDownload struct { - common.KubeAction -} - -func (d *CriDownload) Execute(runtime connector.Runtime) error { - cfg := d.KubeConf.Cluster - archMap := make(map[string]bool) - for _, host := range cfg.Hosts { - switch host.Arch { - case "amd64": - archMap["amd64"] = true - case "arm64": - archMap["arm64"] = true - default: - return errors.New(fmt.Sprintf("Unsupported architecture: %s", host.Arch)) - } - } - - for arch := range archMap { - if err := CriDownloadHTTP(d.KubeConf, runtime.GetWorkDir(), arch, d.PipelineCache); err != nil { - return err - } - } - return nil -} diff --git a/cmd/kk/pkg/bootstrap/confirm/module.go b/cmd/kk/pkg/bootstrap/confirm/module.go deleted file mode 100644 index 5bb67d65..00000000 --- a/cmd/kk/pkg/bootstrap/confirm/module.go +++ /dev/null @@ -1,159 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package confirm - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type InstallConfirmModule struct { - common.KubeModule - Skip bool -} - -func (i *InstallConfirmModule) IsSkip() bool { - return i.Skip -} - -func (i *InstallConfirmModule) Init() { - i.Name = "ConfirmModule" - i.Desc = "Display confirmation form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: new(InstallationConfirm), - } - - i.Tasks = []task.Interface{ - display, - } -} - -type DeleteClusterConfirmModule struct { - common.KubeModule - Skip bool -} - -func (d *DeleteClusterConfirmModule) IsSkip() bool { - return d.Skip -} - -func (d *DeleteClusterConfirmModule) Init() { - d.Name = "DeleteClusterConfirmModule" - d.Desc = "Display delete confirmation form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: &DeleteConfirm{Content: "cluster"}, - } - - d.Tasks = []task.Interface{ - display, - } -} - -type DeleteNodeConfirmModule struct { - common.KubeModule - Skip bool -} - -func (d *DeleteNodeConfirmModule) IsSkip() bool { - return d.Skip -} - -func (d *DeleteNodeConfirmModule) Init() { - d.Name = "DeleteNodeConfirmModule" - d.Desc = "Display delete node confirmation form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: &DeleteConfirm{Content: "node"}, - } - - d.Tasks = []task.Interface{ - display, - } -} - -type UpgradeConfirmModule struct { - common.KubeModule - Skip bool -} - -func (u *UpgradeConfirmModule) IsSkip() bool { - return u.Skip -} - -func (u *UpgradeConfirmModule) Init() { - u.Name = "UpgradeConfirmModule" - u.Desc = "Display upgrade confirmation form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: new(UpgradeConfirm), - } - - u.Tasks = []task.Interface{ - display, - } -} - -type CheckFileExistModule struct { - module.BaseTaskModule - FileName string -} - -func (c *CheckFileExistModule) Init() { - c.Name = "CheckFileExist" - c.Desc = "Check file if is existed" - - check := &task.LocalTask{ - Name: "CheckExist", - Desc: "Check output file if existed", - Action: &CheckFile{FileName: c.FileName}, - } - - c.Tasks = []task.Interface{ - check, - } -} - -type MigrateCriConfirmModule struct { - common.KubeModule -} - -func (d *MigrateCriConfirmModule) Init() { - d.Name = "MigrateCriConfirmModule" - d.Desc = "Display Migrate Cri form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: &MigrateCri{}, - } - - d.Tasks = []task.Interface{ - display, - } - -} diff --git a/cmd/kk/pkg/bootstrap/confirm/tasks.go b/cmd/kk/pkg/bootstrap/confirm/tasks.go deleted file mode 100644 index 06926da0..00000000 --- a/cmd/kk/pkg/bootstrap/confirm/tasks.go +++ /dev/null @@ -1,374 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package confirm - -import ( - "bufio" - "fmt" - "os" - "regexp" - "strings" - - "github.com/mitchellh/mapstructure" - "github.com/modood/table" - "github.com/pkg/errors" - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -// PreCheckResults defines the items to be checked. -type PreCheckResults struct { - Name string `table:"name"` - Sudo string `table:"sudo"` - Curl string `table:"curl"` - Openssl string `table:"openssl"` - Ebtables string `table:"ebtables"` - Socat string `table:"socat"` - Ipset string `table:"ipset"` - Ipvsadm string `table:"ipvsadm"` - Conntrack string `table:"conntrack"` - Chronyd string `table:"chrony"` - Docker string `table:"docker"` - Containerd string `table:"containerd"` - Nfs string `table:"nfs client"` - Ceph string `table:"ceph client"` - Glusterfs string `table:"glusterfs client"` - Time string `table:"time"` -} - -type InstallationConfirm struct { - common.KubeAction -} - -func (i *InstallationConfirm) Execute(runtime connector.Runtime) error { - var ( - results []PreCheckResults - stopFlag bool - ) - - pre := make([]map[string]string, 0, len(runtime.GetAllHosts())) - for _, host := range runtime.GetAllHosts() { - if v, ok := host.GetCache().Get(common.NodePreCheck); ok { - pre = append(pre, v.(map[string]string)) - } else { - return errors.New("get node check result failed by host cache") - } - } - - for node := range pre { - var result PreCheckResults - _ = mapstructure.Decode(pre[node], &result) - results = append(results, result) - } - table.OutputA(results) - reader := bufio.NewReader(os.Stdin) - - if i.KubeConf.Arg.Artifact == "" { - for _, host := range results { - if host.Sudo == "" { - logger.Log.Errorf("%s: sudo is required.", host.Name) - stopFlag = true - } - - if host.Conntrack == "" { - logger.Log.Errorf("%s: conntrack is required.", host.Name) - stopFlag = true - } - - if host.Socat == "" { - logger.Log.Errorf("%s: socat is required.", host.Name) - stopFlag = true - } - } - } - - fmt.Println("") - fmt.Println("This is a simple check of your environment.") - fmt.Println("Before installation, ensure that your machines meet all requirements specified at") - fmt.Println("https://github.com/kubesphere/kubekey#requirements-and-recommendations") - fmt.Println("") - - if k8sVersion, err := versionutil.ParseGeneric(i.KubeConf.Cluster.Kubernetes.Version); err == nil { - if k8sVersion.AtLeast(versionutil.MustParseSemantic("v1.24.0")) && i.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker { - fmt.Println("[Notice]") - fmt.Println("Incorrect runtime. Please specify a container runtime other than Docker to install Kubernetes v1.24 or later.") - fmt.Println("You can set \"spec.kubernetes.containerManager\" in the configuration file to \"containerd\" or add \"--container-manager containerd\" to the \"./kk create cluster\" command.") - fmt.Println("For more information, see:") - fmt.Println("https://github.com/kubesphere/kubekey/blob/master/docs/commands/kk-create-cluster.md") - fmt.Println("https://kubernetes.io/docs/setup/production-environment/container-runtimes/#container-runtimes") - fmt.Println("https://kubernetes.io/blog/2022/02/17/dockershim-faq/") - fmt.Println("") - stopFlag = true - } - } - - if stopFlag { - os.Exit(1) - } - - if i.KubeConf.Arg.SkipConfirmCheck { - return nil - } - - confirmOK := false - for !confirmOK { - fmt.Printf("Continue this installation? [yes/no]: ") - input, err := reader.ReadString('\n') - if err != nil { - logger.Log.Fatal(err) - } - input = strings.TrimSpace(strings.ToLower(input)) - - switch strings.ToLower(input) { - case "yes", "y": - confirmOK = true - case "no", "n": - os.Exit(0) - default: - continue - } - } - return nil -} - -type DeleteConfirm struct { - common.KubeAction - Content string -} - -func (d *DeleteConfirm) Execute(runtime connector.Runtime) error { - reader := bufio.NewReader(os.Stdin) - - confirmOK := false - for !confirmOK { - fmt.Printf("Are you sure to delete this %s? [yes/no]: ", d.Content) - input, err := reader.ReadString('\n') - if err != nil { - return err - } - input = strings.ToLower(strings.TrimSpace(input)) - - switch strings.ToLower(input) { - case "yes", "y": - confirmOK = true - case "no", "n": - os.Exit(0) - default: - continue - } - } - - return nil -} - -type UpgradeConfirm struct { - common.KubeAction -} - -func (u *UpgradeConfirm) Execute(runtime connector.Runtime) error { - pre := make([]map[string]string, len(runtime.GetAllHosts()), len(runtime.GetAllHosts())) - for i, host := range runtime.GetAllHosts() { - if v, ok := host.GetCache().Get(common.NodePreCheck); ok { - pre[i] = v.(map[string]string) - } else { - return errors.New("get node check result failed by host cache") - } - } - - results := make([]PreCheckResults, len(pre), len(pre)) - for i := range pre { - var result PreCheckResults - _ = mapstructure.Decode(pre[i], &result) - results[i] = result - } - table.OutputA(results) - fmt.Println() - - warningFlag := false - cmp, err := versionutil.MustParseSemantic(u.KubeConf.Cluster.Kubernetes.Version).Compare("v1.19.0") - if err != nil { - logger.Log.Fatalf("Failed to compare kubernetes version: %v", err) - } - if cmp == 0 || cmp == 1 { - for _, result := range results { - if len(result.Docker) != 0 { - dockerVersion, err := RefineDockerVersion(result.Docker) - if err != nil { - logger.Log.Fatalf("Failed to get docker version: %v", err) - } - cmp, err := versionutil.MustParseSemantic(dockerVersion).Compare("20.10.0") - if err != nil { - logger.Log.Fatalf("Failed to compare docker version: %v", err) - } - warningFlag = warningFlag || (cmp == -1) - } - } - if warningFlag { - fmt.Println(` -Warning: - - An old Docker version may cause the failure of upgrade. It is recommended that you upgrade Docker to 20.10+ beforehand. - - Issue: https://github.com/kubernetes/kubernetes/issues/101056`) - fmt.Print("\n") - } - } - - nodeStats, ok := u.PipelineCache.GetMustString(common.ClusterNodeStatus) - if !ok { - return errors.New("get cluster nodes status failed by pipeline cache") - } - fmt.Println("Cluster nodes status:") - fmt.Println(nodeStats + "\n") - - fmt.Println("Upgrade Confirmation:") - currentK8sVersion, ok := u.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return errors.New("get current Kubernetes version failed by pipeline cache") - } - fmt.Printf("kubernetes version: %s to %s\n", currentK8sVersion, u.KubeConf.Cluster.Kubernetes.Version) - - if u.KubeConf.Cluster.KubeSphere.Enabled { - currentKsVersion, ok := u.PipelineCache.GetMustString(common.KubeSphereVersion) - if !ok { - return errors.New("get current KubeSphere version failed by pipeline cache") - } - fmt.Printf("kubesphere version: %s to %s\n", currentKsVersion, u.KubeConf.Cluster.KubeSphere.Version) - } - fmt.Println() - - if k8sVersion, err := versionutil.ParseGeneric(u.KubeConf.Cluster.Kubernetes.Version); err == nil { - if cri, ok := u.PipelineCache.GetMustString(common.ClusterNodeCRIRuntimes); ok { - k8sV124 := versionutil.MustParseSemantic("v1.24.0") - if k8sVersion.AtLeast(k8sV124) && versionutil.MustParseSemantic(currentK8sVersion).LessThan(k8sV124) && strings.Contains(cri, "docker") { - fmt.Println("[Notice]") - fmt.Println("Pre-upgrade check failed. The container runtime of the current cluster is Docker.") - fmt.Println("Kubernetes v1.24 and later no longer support dockershim and Docker.") - fmt.Println("Make sure you have completed the migration from Docker to other container runtimes that are compatible with the Kubernetes CRI.") - fmt.Println("For more information, see:") - fmt.Println("https://kubernetes.io/docs/setup/production-environment/container-runtimes/#container-runtimes") - fmt.Println("https://kubernetes.io/blog/2022/02/17/dockershim-faq/") - fmt.Println("") - } - } - } - - reader := bufio.NewReader(os.Stdin) - confirmOK := false - for !confirmOK { - fmt.Printf("Continue upgrading cluster? [yes/no]: ") - input, err := reader.ReadString('\n') - if err != nil { - return err - } - input = strings.ToLower(strings.TrimSpace(input)) - - switch input { - case "yes", "y": - confirmOK = true - case "no", "n": - os.Exit(0) - default: - continue - } - } - return nil -} - -func RefineDockerVersion(version string) (string, error) { - var newVersionComponents []string - versionMatchRE := regexp.MustCompile(`^\s*v?([0-9]+(?:\.[0-9]+)*)(.*)*$`) - parts := versionMatchRE.FindStringSubmatch(version) - if parts == nil { - return "", fmt.Errorf("could not parse %q as version", version) - } - numbers, _ := parts[1], parts[2] - components := strings.Split(numbers, ".") - - for _, c := range components { - newVersion := strings.TrimPrefix(c, "0") - if newVersion == "" { - newVersion = "0" - } - newVersionComponents = append(newVersionComponents, newVersion) - } - return strings.Join(newVersionComponents, "."), nil -} - -type CheckFile struct { - action.BaseAction - FileName string -} - -func (c *CheckFile) Execute(runtime connector.Runtime) error { - if util.IsExist(c.FileName) { - reader := bufio.NewReader(os.Stdin) - stop := false - for { - if stop { - break - } - fmt.Printf("%s already exists. Are you sure you want to overwrite this file? [yes/no]: ", c.FileName) - input, _ := reader.ReadString('\n') - input = strings.ToLower(strings.TrimSpace(input)) - - if input != "" { - switch input { - case "yes", "y": - stop = true - case "no", "n": - os.Exit(0) - } - } - } - } - return nil -} - -type MigrateCri struct { - common.KubeAction -} - -func (d *MigrateCri) Execute(runtime connector.Runtime) error { - reader := bufio.NewReader(os.Stdin) - - confirmOK := false - for !confirmOK { - fmt.Printf("Are you sure to Migrate Cri? [yes/no]: ") - input, err := reader.ReadString('\n') - if err != nil { - return err - } - input = strings.ToLower(strings.TrimSpace(input)) - - switch strings.ToLower(input) { - case "yes", "y": - confirmOK = true - case "no", "n": - os.Exit(0) - default: - continue - } - } - - return nil -} diff --git a/cmd/kk/pkg/bootstrap/confirm/tasks_test.go b/cmd/kk/pkg/bootstrap/confirm/tasks_test.go deleted file mode 100644 index 63c10aa9..00000000 --- a/cmd/kk/pkg/bootstrap/confirm/tasks_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package confirm - -import "testing" - -func TestRefineDockerVersion(t *testing.T) { - cases := map[string]string{ - "1.2.3": "1.2.3", - "23.0.1": "23.0.1", - " 1.2.3 ": "1.2.3", - " 23.0.1 ": "23.0.1", - "abc": "err", - } - - for k, v := range cases { - result, err := RefineDockerVersion(k) - if err != nil { - if v != "err" { - t.Errorf("case \"%s\", result \"%s\" expected, but \"%s\" get", k, v, "err") - } - } else { - if result != v { - t.Errorf("case \"%s\", result \"%s\" expected, but \"%s\" get", k, v, result) - } - } - } -} diff --git a/cmd/kk/pkg/bootstrap/customscripts/module.go b/cmd/kk/pkg/bootstrap/customscripts/module.go deleted file mode 100644 index 1c98a178..00000000 --- a/cmd/kk/pkg/bootstrap/customscripts/module.go +++ /dev/null @@ -1,52 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package customscripts - -import ( - "fmt" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type CustomScriptsModule struct { - module.BaseTaskModule - Phase string - Scripts []kubekeyapiv1alpha2.CustomScripts -} - -func (m *CustomScriptsModule) Init() { - m.Name = fmt.Sprintf("CustomScriptsModule Phase:%s", m.Phase) - m.Desc = "Exec custom shell scripts for each nodes." - - for idx, script := range m.Scripts { - - taskName := fmt.Sprintf("Phase:%s(%d/%d) script:%s", m.Phase, idx, len(m.Scripts), script.Name) - taskDir := fmt.Sprintf("%s-%d-script", m.Phase, idx) - task := &task.RemoteTask{ - Name: taskName, - Desc: taskName, - Hosts: m.Runtime.GetAllHosts(), - Action: &CustomScriptTask{taskDir: taskDir, script: script}, - Parallel: true, - Retry: 1, - } - - m.Tasks = append(m.Tasks, task) - } -} diff --git a/cmd/kk/pkg/bootstrap/customscripts/tasks.go b/cmd/kk/pkg/bootstrap/customscripts/tasks.go deleted file mode 100644 index 439e4dc0..00000000 --- a/cmd/kk/pkg/bootstrap/customscripts/tasks.go +++ /dev/null @@ -1,117 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package customscripts - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type CustomScriptTask struct { - action.BaseAction - taskDir string - script kubekeyapiv1alpha2.CustomScripts -} - -func (t *CustomScriptTask) Execute(runtime connector.Runtime) error { - - if len(t.script.Bash) <= 0 { - return errors.Errorf("custom script %s Bash is empty", t.script.Name) - } - - remoteTaskHome := common.TmpDir + t.taskDir - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mkdir -p %s", remoteTaskHome), false); err != nil { - return errors.Wrapf(err, "create remoteTaskHome: %s err:%s", remoteTaskHome, err) - } - - // dilver the dependency materials to the remotehost - for idx, localPath := range t.script.Materials { - - if !util.IsExist(localPath) { - return errors.Errorf("Not found Path: %s", localPath) - } - - targetPath := filepath.Join(remoteTaskHome, filepath.Base(localPath)) - - // first clean the target to makesure target path always is the lastest. - cleanCmd := fmt.Sprintf("rm -fr %s", targetPath) - if _, err := runtime.GetRunner().SudoCmd(cleanCmd, false); err != nil { - return errors.Wrapf(err, "Can not remove target found Path: %s", targetPath) - } - - start := time.Now() - err := runtime.GetRunner().SudoScp(localPath, targetPath) - if err != nil { - return errors.Wrapf(err, "Can not Scp -fr %s root@%s:%s", localPath, runtime.RemoteHost().GetAddress(), targetPath) - } - - fmt.Printf("Copy %d/%d materials: Scp -fr %s root@%s:%s done, take %s\n", - idx, len(t.script.Materials), localPath, runtime.RemoteHost().GetAddress(), targetPath, time.Since(start)) - } - - // wrap use bash file if shell has many lines. - RunBash := t.script.Bash - if strings.Index(RunBash, "\n") > 0 { - tmpFile, err := os.CreateTemp(os.TempDir(), t.taskDir) - if err != nil { - return errors.Wrapf(err, "create tmp Bash: %s/%s in local node, err:%s", os.TempDir(), t.taskDir, err) - } - defer os.Remove(tmpFile.Name()) - - if _, err := tmpFile.WriteString(RunBash); err != nil { - return errors.Wrapf(err, "write to tmp:%s in local node, err:%s", tmpFile.Name(), err) - } - - targetPath := filepath.Join(remoteTaskHome, "task.sh") - if err := runtime.GetRunner().SudoScp(tmpFile.Name(), targetPath); err != nil { - return errors.Wrapf(err, "Can not Scp -fr %s root@%s:%s", tmpFile.Name(), runtime.RemoteHost().GetAddress(), targetPath) - } - - RunBash = "/bin/bash " + targetPath - } - - start := time.Now() - out, err := runtime.GetRunner().SudoCmd(RunBash, false) - if err != nil { - return errors.Errorf("Exec Bash: %s err:%s", RunBash, err) - } - - if !runtime.GetRunner().Debug { - fmt.Printf("Exec Bash:%s done, take %s", RunBash, time.Since(start)) - cleanCmd := fmt.Sprintf("rm -fr %s", remoteTaskHome) - if _, err := runtime.GetRunner().SudoCmd(cleanCmd, false); err != nil { - return errors.Wrapf(err, "Exec cmd:%s err:%s", cleanCmd, err) - } - } else { - // keep the Materials for debug - fmt.Printf("Exec Bash:%s done, take %s, output:\n%s", RunBash, time.Since(start), out) - } - - return nil -} diff --git a/cmd/kk/pkg/bootstrap/os/common.go b/cmd/kk/pkg/bootstrap/os/common.go deleted file mode 100644 index 59ce5c29..00000000 --- a/cmd/kk/pkg/bootstrap/os/common.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package os - -const ( - Release = "release" -) diff --git a/cmd/kk/pkg/bootstrap/os/module.go b/cmd/kk/pkg/bootstrap/os/module.go deleted file mode 100644 index ef34ae4e..00000000 --- a/cmd/kk/pkg/bootstrap/os/module.go +++ /dev/null @@ -1,352 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package os - -import ( - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type ConfigureOSModule struct { - common.KubeModule - Skip bool -} - -func (c *ConfigureOSModule) IsSkip() bool { - return c.Skip -} - -func (c *ConfigureOSModule) Init() { - c.Name = "ConfigureOSModule" - c.Desc = "Init os dependencies" - - getOSData := &task.RemoteTask{ - Name: "GetOSData", - Desc: "Get OS release", - Hosts: c.Runtime.GetAllHosts(), - Action: new(GetOSData), - Parallel: true, - } - - initOS := &task.RemoteTask{ - Name: "InitOS", - Desc: "Prepare to init OS", - Hosts: c.Runtime.GetAllHosts(), - Action: new(NodeConfigureOS), - Parallel: true, - } - - GenerateScript := &task.RemoteTask{ - Name: "GenerateScript", - Desc: "Generate init os script", - Hosts: c.Runtime.GetAllHosts(), - Action: &action.Template{ - Template: templates.InitOsScriptTmpl, - Dst: filepath.Join(common.KubeScriptDir, "initOS.sh"), - Data: util.Data{ - "Hosts": templates.GenerateHosts(c.Runtime, c.KubeConf), - }, - }, - Parallel: true, - } - - ExecScript := &task.RemoteTask{ - Name: "ExecScript", - Desc: "Exec init os script", - Hosts: c.Runtime.GetAllHosts(), - Action: new(NodeExecScript), - Parallel: true, - } - - ConfigureNtpServer := &task.RemoteTask{ - Name: "ConfigureNtpServer", - Desc: "configure the ntp server for each node", - Hosts: c.Runtime.GetAllHosts(), - Prepare: new(NodeConfigureNtpCheck), - Action: new(NodeConfigureNtpServer), - Parallel: true, - } - - c.Tasks = []task.Interface{ - getOSData, - initOS, - GenerateScript, - ExecScript, - ConfigureNtpServer, - } -} - -type ClearNodeOSModule struct { - common.KubeModule -} - -func (c *ClearNodeOSModule) Init() { - c.Name = "ClearNodeOSModule" - - stopKubelet := &task.RemoteTask{ - Name: "StopKubelet", - Desc: "Stop Kubelet", - Hosts: c.Runtime.GetHostsByRole(common.Worker), - Prepare: new(DeleteNode), - Action: new(StopKubelet), - Parallel: true, - } - - resetNetworkConfig := &task.RemoteTask{ - Name: "ResetNetworkConfig", - Desc: "Reset os network config", - Hosts: c.Runtime.GetHostsByRole(common.Worker), - Prepare: new(DeleteNode), - Action: new(ResetNetworkConfig), - Parallel: true, - } - - removeFiles := &task.RemoteTask{ - Name: "RemoveFiles", - Desc: "Remove node files", - Hosts: c.Runtime.GetHostsByRole(common.Worker), - Prepare: new(DeleteNode), - Action: new(RemoveNodeFiles), - Parallel: true, - } - - daemonReload := &task.RemoteTask{ - Name: "DaemonReload", - Desc: "Systemd daemon reload", - Hosts: c.Runtime.GetHostsByRole(common.Worker), - Prepare: new(DeleteNode), - Action: new(DaemonReload), - Parallel: true, - } - - c.Tasks = []task.Interface{ - stopKubelet, - resetNetworkConfig, - removeFiles, - daemonReload, - } -} - -type ClearOSEnvironmentModule struct { - common.KubeModule -} - -func (c *ClearOSEnvironmentModule) Init() { - c.Name = "ClearOSModule" - - resetNetworkConfig := &task.RemoteTask{ - Name: "ResetNetworkConfig", - Desc: "Reset os network config", - Hosts: c.Runtime.GetHostsByRole(common.K8s), - Action: new(ResetNetworkConfig), - Parallel: true, - } - - uninstallETCD := &task.RemoteTask{ - Name: "UninstallETCD", - Desc: "Uninstall etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: &prepare.PrepareCollection{ - new(EtcdTypeIsKubeKey), - }, - Action: new(UninstallETCD), - Parallel: true, - } - - removeFiles := &task.RemoteTask{ - Name: "RemoveFiles", - Desc: "Remove cluster files", - Hosts: c.Runtime.GetHostsByRole(common.K8s), - Action: new(RemoveFiles), - Parallel: true, - } - - daemonReload := &task.RemoteTask{ - Name: "DaemonReload", - Desc: "Systemd daemon reload", - Hosts: c.Runtime.GetHostsByRole(common.K8s), - Action: new(DaemonReload), - Parallel: true, - } - - c.Tasks = []task.Interface{ - resetNetworkConfig, - uninstallETCD, - removeFiles, - daemonReload, - } -} - -type RepositoryOnlineModule struct { - common.KubeModule - Skip bool -} - -func (r *RepositoryOnlineModule) IsSkip() bool { - return r.Skip -} - -func (r *RepositoryOnlineModule) Init() { - r.Name = "RepositoryOnlineModule" - - getOSData := &task.RemoteTask{ - Name: "GetOSData", - Desc: "Get OS release", - Hosts: r.Runtime.GetAllHosts(), - Action: new(GetOSData), - Parallel: true, - } - - newRepo := &task.RemoteTask{ - Name: "NewRepoClient", - Desc: "New repository client", - Hosts: r.Runtime.GetAllHosts(), - Action: new(NewRepoClient), - Parallel: true, - Retry: 1, - } - - install := &task.RemoteTask{ - Name: "InstallPackage", - Desc: "Install packages", - Hosts: r.Runtime.GetAllHosts(), - Action: new(InstallPackage), - Parallel: true, - Retry: 1, - } - - r.Tasks = []task.Interface{ - getOSData, - newRepo, - install, - } -} - -type RepositoryModule struct { - common.KubeModule - Skip bool -} - -func (r *RepositoryModule) IsSkip() bool { - return r.Skip -} - -func (r *RepositoryModule) Init() { - r.Name = "RepositoryModule" - r.Desc = "Install local repository" - - getOSData := &task.RemoteTask{ - Name: "GetOSData", - Desc: "Get OS release", - Hosts: r.Runtime.GetAllHosts(), - Action: new(GetOSData), - Parallel: true, - } - - sync := &task.RemoteTask{ - Name: "SyncRepositoryISOFile", - Desc: "Sync repository iso file to all nodes", - Hosts: r.Runtime.GetAllHosts(), - Action: new(SyncRepositoryFile), - Parallel: true, - Retry: 2, - } - - mount := &task.RemoteTask{ - Name: "MountISO", - Desc: "Mount iso file", - Hosts: r.Runtime.GetAllHosts(), - Action: new(MountISO), - Parallel: true, - Retry: 1, - } - - newRepo := &task.RemoteTask{ - Name: "NewRepoClient", - Desc: "New repository client", - Hosts: r.Runtime.GetAllHosts(), - Action: new(NewRepoClient), - Parallel: true, - Retry: 1, - Rollback: new(RollbackUmount), - } - - backup := &task.RemoteTask{ - Name: "BackupOriginalRepository", - Desc: "Backup original repository", - Hosts: r.Runtime.GetAllHosts(), - Action: new(BackupOriginalRepository), - Parallel: true, - Retry: 1, - Rollback: new(RecoverBackupSuccessNode), - } - - add := &task.RemoteTask{ - Name: "AddLocalRepository", - Desc: "Add local repository", - Hosts: r.Runtime.GetAllHosts(), - Action: new(AddLocalRepository), - Parallel: true, - Retry: 1, - Rollback: new(RecoverRepository), - } - - install := &task.RemoteTask{ - Name: "InstallPackage", - Desc: "Install packages", - Hosts: r.Runtime.GetAllHosts(), - Action: new(InstallPackage), - Parallel: true, - Retry: 1, - Rollback: new(RecoverRepository), - } - - reset := &task.RemoteTask{ - Name: "ResetRepository", - Desc: "Reset repository to the original repository", - Hosts: r.Runtime.GetAllHosts(), - Action: new(ResetRepository), - Parallel: true, - Retry: 1, - } - - umount := &task.RemoteTask{ - Name: "UmountISO", - Desc: "Umount ISO file", - Hosts: r.Runtime.GetAllHosts(), - Action: new(UmountISO), - Parallel: true, - } - - r.Tasks = []task.Interface{ - getOSData, - sync, - mount, - newRepo, - backup, - add, - install, - reset, - umount, - } -} diff --git a/cmd/kk/pkg/bootstrap/os/prepares.go b/cmd/kk/pkg/bootstrap/os/prepares.go deleted file mode 100644 index 67900413..00000000 --- a/cmd/kk/pkg/bootstrap/os/prepares.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package os - -import ( - kubekeyv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type NodeConfigureNtpCheck struct { - common.KubePrepare -} - -func (n *NodeConfigureNtpCheck) PreCheck(_ connector.Runtime) (bool, error) { - // skip when both NtpServers and Timezone was not set in cluster config - if len(n.KubeConf.Cluster.System.NtpServers) == 0 && len(n.KubeConf.Cluster.System.Timezone) == 0 { - return false, nil - } - - return true, nil -} - -type EtcdTypeIsKubeKey struct { - common.KubePrepare -} - -func (e *EtcdTypeIsKubeKey) PreCheck(_ connector.Runtime) (bool, error) { - if len(e.KubeConf.Cluster.Etcd.Type) == 0 || e.KubeConf.Cluster.Etcd.Type == kubekeyv1alpha2.KubeKey { - return true, nil - } - - return false, nil -} - -type DeleteNode struct { - common.KubePrepare -} - -func (d *DeleteNode) PreCheck(runtime connector.Runtime) (bool, error) { - nodeName, ok := d.PipelineCache.Get("dstNode") - if !ok { - return true, nil - } - - host := runtime.RemoteHost() - if host.GetName() == nodeName { - return true, nil - } - - return false, nil -} diff --git a/cmd/kk/pkg/bootstrap/os/repository/repository.go b/cmd/kk/pkg/bootstrap/os/repository/repository.go deleted file mode 100644 index 89ae3de5..00000000 --- a/cmd/kk/pkg/bootstrap/os/repository/repository.go +++ /dev/null @@ -1,44 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package repository - -import ( - "fmt" - "strings" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type Interface interface { - Backup(runtime connector.Runtime) error - IsAlreadyBackUp() bool - Add(runtime connector.Runtime, path string) error - Update(runtime connector.Runtime) error - Install(runtime connector.Runtime, pkg ...string) error - Reset(runtime connector.Runtime) error -} - -func New(os string) (Interface, error) { - switch strings.ToLower(os) { - case "ubuntu", "debian": - return NewDeb(), nil - case "centos", "rhel": - return NewRPM(), nil - default: - return nil, fmt.Errorf("unsupported operation system %s", os) - } -} diff --git a/cmd/kk/pkg/bootstrap/os/repository/repository_deb.go b/cmd/kk/pkg/bootstrap/os/repository/repository_deb.go deleted file mode 100644 index f535fe78..00000000 --- a/cmd/kk/pkg/bootstrap/os/repository/repository_deb.go +++ /dev/null @@ -1,106 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package repository - -import ( - "fmt" - "strings" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type Debian struct { - backup bool -} - -func NewDeb() Interface { - return &Debian{} -} - -func (d *Debian) Backup(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("mv /etc/apt/sources.list /etc/apt/sources.list.kubekey.bak", false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd("mv /etc/apt/sources.list.d /etc/apt/sources.list.d.kubekey.bak", false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd("mkdir -p /etc/apt/sources.list.d", false); err != nil { - return err - } - d.backup = true - return nil -} - -func (d *Debian) IsAlreadyBackUp() bool { - return d.backup -} - -func (d *Debian) Add(runtime connector.Runtime, path string) error { - if !d.IsAlreadyBackUp() { - return fmt.Errorf("linux repository must be backuped before") - } - - if _, err := runtime.GetRunner().SudoCmd("rm -rf /etc/apt/sources.list.d/*", false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("echo 'deb [trusted=yes] file://%s /' > /etc/apt/sources.list.d/kubekey.list", path), - true); err != nil { - return err - } - return nil -} - -func (d *Debian) Update(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().Cmd("sudo apt-get update", true); err != nil { - return err - } - return nil -} - -func (d *Debian) Install(runtime connector.Runtime, pkg ...string) error { - defaultPkg := []string{"socat", "conntrack", "ipset", "ebtables", "chrony", "ipvsadm"} - if len(pkg) == 0 { - pkg = defaultPkg - } else { - pkg = append(pkg, defaultPkg...) - } - - str := strings.Join(pkg, " ") - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("apt install -y %s", str), true); err != nil { - return err - } - return nil -} - -func (d *Debian) Reset(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("rm -rf /etc/apt/sources.list.d", false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd("mv /etc/apt/sources.list.kubekey.bak /etc/apt/sources.list", false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd("mv /etc/apt/sources.list.d.kubekey.bak /etc/apt/sources.list.d", false); err != nil { - return err - } - - return nil -} diff --git a/cmd/kk/pkg/bootstrap/os/repository/repository_rpm.go b/cmd/kk/pkg/bootstrap/os/repository/repository_rpm.go deleted file mode 100644 index 16606256..00000000 --- a/cmd/kk/pkg/bootstrap/os/repository/repository_rpm.go +++ /dev/null @@ -1,110 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package repository - -import ( - "fmt" - "strings" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type RedhatPackageManager struct { - backup bool -} - -func NewRPM() Interface { - return &RedhatPackageManager{} -} - -func (r *RedhatPackageManager) Backup(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("mv /etc/yum.repos.d /etc/yum.repos.d.kubekey.bak", false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd("mkdir -p /etc/yum.repos.d", false); err != nil { - return err - } - r.backup = true - return nil -} - -func (r *RedhatPackageManager) IsAlreadyBackUp() bool { - return r.backup -} - -func (r *RedhatPackageManager) Add(runtime connector.Runtime, path string) error { - if !r.IsAlreadyBackUp() { - return fmt.Errorf("linux repository must be backuped before") - } - - if _, err := runtime.GetRunner().SudoCmd("rm -rf /etc/yum.repos.d/*", false); err != nil { - return err - } - - content := fmt.Sprintf(`cat << EOF > /etc/yum.repos.d/CentOS-local.repo -[base-local] -name=rpms-local - -baseurl=file://%s - -enabled=1 - -gpgcheck=0 - -EOF -`, path) - if _, err := runtime.GetRunner().SudoCmd(content, false); err != nil { - return err - } - - return nil -} - -func (r *RedhatPackageManager) Update(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("yum clean all && yum makecache", true); err != nil { - return err - } - return nil -} - -func (r *RedhatPackageManager) Install(runtime connector.Runtime, pkg ...string) error { - defaultPkg := []string{"openssl", "socat", "conntrack", "ipset", "ebtables", "chrony", "ipvsadm"} - if len(pkg) == 0 { - pkg = defaultPkg - } else { - pkg = append(pkg, defaultPkg...) - } - - str := strings.Join(pkg, " ") - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("yum install -y %s", str), true); err != nil { - return err - } - return nil -} - -func (r *RedhatPackageManager) Reset(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("rm -rf /etc/yum.repos.d", false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd("mv /etc/yum.repos.d.kubekey.bak /etc/yum.repos.d", false); err != nil { - return err - } - - return nil -} diff --git a/cmd/kk/pkg/bootstrap/os/rollback.go b/cmd/kk/pkg/bootstrap/os/rollback.go deleted file mode 100644 index d6eb6655..00000000 --- a/cmd/kk/pkg/bootstrap/os/rollback.go +++ /dev/null @@ -1,90 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package os - -import ( - "fmt" - "path/filepath" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os/repository" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" -) - -type RollbackUmount struct { - common.KubeRollback -} - -func (r *RollbackUmount) Execute(runtime connector.Runtime, result *ending.ActionResult) error { - mountPath := filepath.Join(common.TmpDir, "iso") - umountCmd := fmt.Sprintf("umount %s", mountPath) - if _, err := runtime.GetRunner().SudoCmd(umountCmd, false); err != nil { - return errors.Wrapf(errors.WithStack(err), "umount %s failed", mountPath) - } - return nil -} - -type RecoverBackupSuccessNode struct { - common.KubeRollback -} - -func (r *RecoverBackupSuccessNode) Execute(runtime connector.Runtime, result *ending.ActionResult) error { - if result.Status == ending.SUCCESS { - host := runtime.RemoteHost() - repo, ok := host.GetCache().Get("repo") - if !ok { - return errors.New("get repo failed by host cache") - } - - re := repo.(repository.Interface) - if err := re.Reset(runtime); err != nil { - return errors.Wrapf(errors.WithStack(err), "reset repository failed") - } - } - - mountPath := filepath.Join(common.TmpDir, "iso") - umountCmd := fmt.Sprintf("umount %s", mountPath) - if _, err := runtime.GetRunner().SudoCmd(umountCmd, false); err != nil { - return errors.Wrapf(errors.WithStack(err), "umount %s failed", mountPath) - } - return nil -} - -type RecoverRepository struct { - common.KubeRollback -} - -func (r *RecoverRepository) Execute(runtime connector.Runtime, result *ending.ActionResult) error { - host := runtime.RemoteHost() - repo, ok := host.GetCache().Get("repo") - if !ok { - return errors.New("get repo failed by host cache") - } - - re := repo.(repository.Interface) - _ = re.Reset(runtime) - - mountPath := filepath.Join(common.TmpDir, "iso") - umountCmd := fmt.Sprintf("umount %s", mountPath) - if _, err := runtime.GetRunner().SudoCmd(umountCmd, false); err != nil { - return errors.Wrapf(errors.WithStack(err), "umount %s failed", mountPath) - } - return nil -} diff --git a/cmd/kk/pkg/bootstrap/os/tasks.go b/cmd/kk/pkg/bootstrap/os/tasks.go deleted file mode 100644 index 1f346761..00000000 --- a/cmd/kk/pkg/bootstrap/os/tasks.go +++ /dev/null @@ -1,559 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package os - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os/repository" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" - "github.com/kubesphere/kubekey/v3/util/osrelease" -) - -type NodeConfigureOS struct { - common.KubeAction -} - -func (n *NodeConfigureOS) Execute(runtime connector.Runtime) error { - - host := runtime.RemoteHost() - if err := addUsers(runtime, host); err != nil { - return errors.Wrap(errors.WithStack(err), "Failed to add users") - } - - if err := createDirectories(runtime, host); err != nil { - return err - } - - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - _, err1 := runtime.GetRunner().SudoCmd(fmt.Sprintf("hostnamectl set-hostname %s && sed -i '/^127.0.1.1/s/.*/127.0.1.1 %s/g' /etc/hosts", host.GetName(), host.GetName()), false) - if err1 != nil { - return errors.Wrap(errors.WithStack(err1), "Failed to override hostname") - } - - return nil -} - -func addUsers(runtime connector.Runtime, node connector.Host) error { - if _, err := runtime.GetRunner().SudoCmd("useradd -M -c 'Kubernetes user' -s /sbin/nologin -r kube || :", false); err != nil { - return err - } - - if node.IsRole(common.ETCD) { - if _, err := runtime.GetRunner().SudoCmd("useradd -M -c 'Etcd user' -s /sbin/nologin -r etcd || :", false); err != nil { - return err - } - } - - return nil -} - -func createDirectories(runtime connector.Runtime, node connector.Host) error { - dirs := []string{ - common.BinDir, - common.KubeConfigDir, - common.KubeCertDir, - common.KubeManifestDir, - common.KubeScriptDir, - common.KubeletFlexvolumesPluginsDir, - } - - for _, dir := range dirs { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mkdir -p %s", dir), false); err != nil { - return err - } - if dir == common.KubeletFlexvolumesPluginsDir { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chown kube -R %s", "/usr/libexec/kubernetes"), false); err != nil { - return err - } - } else { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chown kube -R %s", dir), false); err != nil { - return err - } - } - } - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mkdir -p %s && chown kube -R %s", "/etc/cni/net.d", "/etc/cni"), false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mkdir -p %s && chown kube -R %s", "/opt/cni/bin", "/opt/cni"), false); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mkdir -p %s && chown kube -R %s", "/var/lib/calico", "/var/lib/calico"), false); err != nil { - return err - } - - if node.IsRole(common.ETCD) { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mkdir -p %s && chown etcd -R %s", "/var/lib/etcd", "/var/lib/etcd"), false); err != nil { - return err - } - } - - return nil -} - -type NodeExecScript struct { - common.KubeAction -} - -func (n *NodeExecScript) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s/initOS.sh", common.KubeScriptDir), false); err != nil { - return errors.Wrap(errors.WithStack(err), "Failed to chmod +x init os script") - } - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("%s/initOS.sh", common.KubeScriptDir), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Failed to configure operating system") - } - return nil -} - -var ( - etcdFiles = []string{ - "/usr/local/bin/etcd", - "/etc/ssl/etcd", - "/var/lib/etcd/*", - "/etc/etcd.env", - } - clusterFiles = []string{ - "/etc/kubernetes", - "/etc/systemd/system/etcd.service", - "/etc/systemd/system/backup-etcd.service", - "/etc/systemd/system/backup-etcd.timer", - "/var/log/calico", - "/etc/cni", - "/var/log/pods/", - "/var/lib/cni", - "/var/lib/calico", - "/var/lib/kubelet/*", - "/run/calico", - "/run/flannel", - "/etc/flannel", - "/var/openebs", - "/etc/systemd/system/kubelet.service", - "/etc/systemd/system/kubelet.service.d", - "/usr/local/bin/kubelet", - "/usr/local/bin/kubeadm", - "/usr/bin/kubelet", - "/var/lib/rook", - "/tmp/kubekey", - "/etc/kubekey", - } - - networkResetCmds = []string{ - "iptables -F", - "iptables -X", - "iptables -F -t nat", - "iptables -X -t nat", - "ipvsadm -C", - "ip link del kube-ipvs0", - "ip link del nodelocaldns", - "ip link del cni0", - "ip link del flannel.1", - "ip link del flannel-v6.1", - "ip link del flannel-wg", - "ip link del flannel-wg-v6", - "ip link del cilium_host", - "ip link del cilium_vxlan", - "ip link del vxlan.calico", - "ip link del vxlan-v6.calico", - "ip -br link show | grep 'cali[a-f0-9]*' | awk -F '@' '{print $1}' | xargs -r -t -n 1 ip link del", - "ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns del", - } -) - -type ResetNetworkConfig struct { - common.KubeAction -} - -func (r *ResetNetworkConfig) Execute(runtime connector.Runtime) error { - for _, cmd := range networkResetCmds { - _, _ = runtime.GetRunner().SudoCmd(cmd, true) - } - return nil -} - -type StopKubelet struct { - common.KubeAction -} - -func (s *StopKubelet) Execute(runtime connector.Runtime) error { - _, _ = runtime.GetRunner().SudoCmd("systemctl disable kubelet && systemctl stop kubelet && exit 0", false) - return nil -} - -type UninstallETCD struct { - common.KubeAction -} - -func (s *UninstallETCD) Execute(runtime connector.Runtime) error { - _, _ = runtime.GetRunner().SudoCmd("systemctl stop etcd && exit 0", false) - for _, file := range etcdFiles { - _, _ = runtime.GetRunner().SudoCmd(fmt.Sprintf("rm -rf %s", file), true) - } - return nil -} - -type RemoveNodeFiles struct { - common.KubeAction -} - -func (r *RemoveNodeFiles) Execute(runtime connector.Runtime) error { - nodeFiles := []string{ - "/etc/kubernetes", - "/etc/systemd/system/etcd.service", - "/var/log/calico", - "/etc/cni", - "/var/log/pods/", - "/var/lib/cni", - "/var/lib/calico", - "/var/lib/kubelet/*", - "/run/calico", - "/run/flannel", - "/etc/flannel", - "/etc/systemd/system/kubelet.service", - "/etc/systemd/system/kubelet.service.d", - "/usr/local/bin/kubelet", - "/usr/local/bin/kubeadm", - "/usr/bin/kubelet", - "/tmp/kubekey", - "/etc/kubekey", - } - - for _, file := range nodeFiles { - _, _ = runtime.GetRunner().SudoCmd(fmt.Sprintf("rm -rf %s", file), true) - } - return nil -} - -type RemoveFiles struct { - common.KubeAction -} - -func (r *RemoveFiles) Execute(runtime connector.Runtime) error { - for _, file := range clusterFiles { - _, _ = runtime.GetRunner().SudoCmd(fmt.Sprintf("rm -rf %s", file), true) - } - return nil -} - -type DaemonReload struct { - common.KubeAction -} - -func (d *DaemonReload) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && exit 0", false); err != nil { - return errors.Wrap(errors.WithStack(err), "systemctl daemon-reload failed") - } - - // try to restart the cotainerd after /etc/cni has been removed - _, _ = runtime.GetRunner().SudoCmd("systemctl restart containerd", false) - return nil -} - -type GetOSData struct { - common.KubeAction -} - -func (g *GetOSData) Execute(runtime connector.Runtime) error { - osReleaseStr, err := runtime.GetRunner().SudoCmd("cat /etc/os-release", false) - if err != nil { - return err - } - osrData := osrelease.Parse(strings.Replace(osReleaseStr, "\r\n", "\n", -1)) - - host := runtime.RemoteHost() - // type: *osrelease.data - host.GetCache().Set(Release, osrData) - return nil -} - -type SyncRepositoryFile struct { - common.KubeAction -} - -func (s *SyncRepositoryFile) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return errors.Wrap(err, "reset tmp dir failed") - } - - host := runtime.RemoteHost() - release, ok := host.GetCache().Get(Release) - if !ok { - return errors.New("get os release failed by root cache") - } - r := release.(*osrelease.Data) - - fileName := fmt.Sprintf("%s-%s-%s.iso", r.ID, r.VersionID, host.GetArch()) - src := filepath.Join(runtime.GetWorkDir(), "repository", host.GetArch(), r.ID, r.VersionID, fileName) - dst := filepath.Join(common.TmpDir, fileName) - if err := runtime.GetRunner().Scp(src, dst); err != nil { - return errors.Wrapf(errors.WithStack(err), "scp %s to %s failed", src, dst) - } - - host.GetCache().Set("iso", fileName) - return nil -} - -type MountISO struct { - common.KubeAction -} - -func (m *MountISO) Execute(runtime connector.Runtime) error { - mountPath := filepath.Join(common.TmpDir, "iso") - if err := runtime.GetRunner().MkDir(mountPath); err != nil { - return errors.Wrapf(errors.WithStack(err), "create mount dir failed") - } - - host := runtime.RemoteHost() - isoFile, _ := host.GetCache().GetMustString("iso") - path := filepath.Join(common.TmpDir, isoFile) - mountCmd := fmt.Sprintf("sudo mount -t iso9660 -o loop %s %s", path, mountPath) - if _, err := runtime.GetRunner().Cmd(mountCmd, false); err != nil { - return errors.Wrapf(errors.WithStack(err), "mount %s at %s failed", path, mountPath) - } - return nil -} - -type NewRepoClient struct { - common.KubeAction -} - -func (n *NewRepoClient) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - release, ok := host.GetCache().Get(Release) - if !ok { - return errors.New("get os release failed by host cache") - } - r := release.(*osrelease.Data) - - repo, err := repository.New(r.ID) - if err != nil { - checkDeb, debErr := runtime.GetRunner().SudoCmd("which apt", false) - if debErr == nil && strings.Contains(checkDeb, "bin") { - repo = repository.NewDeb() - } - checkRPM, rpmErr := runtime.GetRunner().SudoCmd("which yum", false) - if rpmErr == nil && strings.Contains(checkRPM, "bin") { - repo = repository.NewRPM() - } - - if debErr != nil && rpmErr != nil { - return errors.Wrap(errors.WithStack(err), "new repository manager failed") - } else if debErr == nil && rpmErr == nil { - return errors.New("can't detect the main package repository, only one of apt or yum is supported") - } - } - - host.GetCache().Set("repo", repo) - return nil -} - -type BackupOriginalRepository struct { - common.KubeAction -} - -func (b *BackupOriginalRepository) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - r, ok := host.GetCache().Get("repo") - if !ok { - return errors.New("get repo failed by host cache") - } - repo := r.(repository.Interface) - - if err := repo.Backup(runtime); err != nil { - return errors.Wrap(errors.WithStack(err), "backup repository failed") - } - - return nil -} - -type AddLocalRepository struct { - common.KubeAction -} - -func (a *AddLocalRepository) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - r, ok := host.GetCache().Get("repo") - if !ok { - return errors.New("get repo failed by host cache") - } - repo := r.(repository.Interface) - - if installErr := repo.Add(runtime, filepath.Join(common.TmpDir, "iso")); installErr != nil { - return errors.Wrap(errors.WithStack(installErr), "add local repository failed") - } - if installErr := repo.Update(runtime); installErr != nil { - return errors.Wrap(errors.WithStack(installErr), "update local repository failed") - } - - return nil -} - -type InstallPackage struct { - common.KubeAction -} - -func (i *InstallPackage) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - repo, ok := host.GetCache().Get("repo") - if !ok { - return errors.New("get repo failed by host cache") - } - r := repo.(repository.Interface) - - var pkg []string - if _, ok := r.(*repository.Debian); ok { - pkg = i.KubeConf.Cluster.System.Debs - } else if _, ok := r.(*repository.RedhatPackageManager); ok { - pkg = i.KubeConf.Cluster.System.Rpms - } - - if installErr := r.Update(runtime); installErr != nil { - return errors.Wrap(errors.WithStack(installErr), "update repository failed") - } - - if installErr := r.Install(runtime, pkg...); installErr != nil { - return errors.Wrap(errors.WithStack(installErr), "install repository package failed") - } - return nil -} - -type ResetRepository struct { - common.KubeAction -} - -func (r *ResetRepository) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - repo, ok := host.GetCache().Get("repo") - if !ok { - return errors.New("get repo failed by host cache") - } - re := repo.(repository.Interface) - - var resetErr error - defer func() { - if resetErr != nil { - mountPath := filepath.Join(common.TmpDir, "iso") - umountCmd := fmt.Sprintf("umount %s", mountPath) - _, _ = runtime.GetRunner().SudoCmd(umountCmd, false) - } - }() - - if resetErr = re.Reset(runtime); resetErr != nil { - return errors.Wrap(errors.WithStack(resetErr), "reset repository failed") - } - - return nil -} - -type UmountISO struct { - common.KubeAction -} - -func (u *UmountISO) Execute(runtime connector.Runtime) error { - mountPath := filepath.Join(common.TmpDir, "iso") - umountCmd := fmt.Sprintf("umount %s", mountPath) - if _, err := runtime.GetRunner().SudoCmd(umountCmd, false); err != nil { - return errors.Wrapf(errors.WithStack(err), "umount %s failed", mountPath) - } - return nil -} - -type NodeConfigureNtpServer struct { - common.KubeAction -} - -func (n *NodeConfigureNtpServer) Execute(runtime connector.Runtime) error { - - currentHost := runtime.RemoteHost() - release, ok := currentHost.GetCache().Get(Release) - if !ok { - return errors.New("get os release failed by host cache") - } - r := release.(*osrelease.Data) - - chronyConfigFile := "/etc/chrony.conf" - chronyService := "chronyd.service" - if r.ID == "ubuntu" || r.ID == "debian" { - chronyConfigFile = "/etc/chrony/chrony.conf" - chronyService = "chrony.service" - } - - // if NtpServers was configured - for _, server := range n.KubeConf.Cluster.System.NtpServers { - - serverAddr := strings.Trim(server, " \"") - if serverAddr == currentHost.GetName() || serverAddr == currentHost.GetInternalAddress() { - allowClientCmd := fmt.Sprintf(`sed -i '/#allow/ a\allow 0.0.0.0/0' %s`, chronyConfigFile) - if _, err := runtime.GetRunner().SudoCmd(allowClientCmd, false); err != nil { - return errors.Wrapf(err, "change host:%s chronyd conf failed, please check file %s", serverAddr, chronyConfigFile) - } - } - - // use internal ip to client chronyd server - for _, host := range runtime.GetAllHosts() { - if serverAddr == host.GetName() { - serverAddr = host.GetInternalAddress() - break - } - } - - checkOrAddCmd := fmt.Sprintf(`grep -q '^server %s iburst' %s||sed '1a server %s iburst' -i %s`, serverAddr, chronyConfigFile, serverAddr, chronyConfigFile) - if _, err := runtime.GetRunner().SudoCmd(checkOrAddCmd, false); err != nil { - return errors.Wrapf(err, "set ntpserver: %s failed, please check file %s", serverAddr, chronyConfigFile) - } - - } - - // if Timezone was configured - if len(n.KubeConf.Cluster.System.Timezone) > 0 { - setTimeZoneCmd := fmt.Sprintf("timedatectl set-timezone %s", n.KubeConf.Cluster.System.Timezone) - if _, err := runtime.GetRunner().SudoCmd(setTimeZoneCmd, false); err != nil { - return errors.Wrapf(err, "set timezone: %s failed", n.KubeConf.Cluster.System.Timezone) - } - - if _, err := runtime.GetRunner().SudoCmd("timedatectl set-ntp true", false); err != nil { - return errors.Wrap(err, "timedatectl set-ntp true failed") - } - } - - // ensure chronyd was enabled and work normally - if len(n.KubeConf.Cluster.System.NtpServers) > 0 || len(n.KubeConf.Cluster.System.Timezone) > 0 { - startChronyCmd := fmt.Sprintf("systemctl enable %s && systemctl restart %s", chronyService, chronyService) - if _, err := runtime.GetRunner().SudoCmd(startChronyCmd, false); err != nil { - return errors.Wrap(err, "restart chronyd failed") - } - - // tells chronyd to cancel any remaining correction that was being slewed and jump the system clock by the equivalent amount, making it correct immediately. - if _, err := runtime.GetRunner().SudoCmd("chronyc makestep > /dev/null && chronyc sources", true); err != nil { - return errors.Wrap(err, "chronyc makestep failed") - } - } - - return nil -} diff --git a/cmd/kk/pkg/bootstrap/os/templates/init_script.go b/cmd/kk/pkg/bootstrap/os/templates/init_script.go deleted file mode 100644 index 5bceb208..00000000 --- a/cmd/kk/pkg/bootstrap/os/templates/init_script.go +++ /dev/null @@ -1,247 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "fmt" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry" - "text/template" - - "github.com/lithammer/dedent" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -var InitOsScriptTmpl = template.Must(template.New("initOS.sh").Parse( - dedent.Dedent(`#!/usr/bin/env bash - -# Copyright 2020 The KubeSphere Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -swapoff -a -sed -i /^[^#]*swap*/s/^/\#/g /etc/fstab - -# See https://github.com/kubernetes/website/issues/14457 -if [ -f /etc/selinux/config ]; then - sed -ri 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config -fi -# for ubuntu: sudo apt install selinux-utils -# for centos: yum install selinux-policy -if command -v setenforce &> /dev/null -then - setenforce 0 - getenforce -fi - -echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf -echo 'net.bridge.bridge-nf-call-arptables = 1' >> /etc/sysctl.conf -echo 'net.bridge.bridge-nf-call-ip6tables = 1' >> /etc/sysctl.conf -echo 'net.bridge.bridge-nf-call-iptables = 1' >> /etc/sysctl.conf -echo 'net.ipv4.ip_local_reserved_ports = 30000-32767' >> /etc/sysctl.conf -echo 'net.core.netdev_max_backlog = 65535' >> /etc/sysctl.conf -echo 'net.core.rmem_max = 33554432' >> /etc/sysctl.conf -echo 'net.core.wmem_max = 33554432' >> /etc/sysctl.conf -echo 'net.core.somaxconn = 32768' >> /etc/sysctl.conf -echo 'net.ipv4.tcp_max_syn_backlog = 1048576' >> /etc/sysctl.conf -echo 'net.ipv4.neigh.default.gc_thresh1 = 512' >> /etc/sysctl.conf -echo 'net.ipv4.neigh.default.gc_thresh2 = 2048' >> /etc/sysctl.conf -echo 'net.ipv4.neigh.default.gc_thresh3 = 4096' >> /etc/sysctl.conf -echo 'net.ipv4.tcp_retries2 = 15' >> /etc/sysctl.conf -echo 'net.ipv4.tcp_max_tw_buckets = 1048576' >> /etc/sysctl.conf -echo 'net.ipv4.tcp_max_orphans = 65535' >> /etc/sysctl.conf -echo 'net.ipv4.udp_rmem_min = 131072' >> /etc/sysctl.conf -echo 'net.ipv4.udp_wmem_min = 131072' >> /etc/sysctl.conf -echo 'net.ipv4.conf.all.rp_filter = 1' >> /etc/sysctl.conf -echo 'net.ipv4.conf.default.rp_filter = 1' >> /etc/sysctl.conf -echo 'net.ipv4.conf.all.arp_accept = 1' >> /etc/sysctl.conf -echo 'net.ipv4.conf.default.arp_accept = 1' >> /etc/sysctl.conf -echo 'net.ipv4.conf.all.arp_ignore = 1' >> /etc/sysctl.conf -echo 'net.ipv4.conf.default.arp_ignore = 1' >> /etc/sysctl.conf -echo 'vm.max_map_count = 262144' >> /etc/sysctl.conf -echo 'vm.swappiness = 0' >> /etc/sysctl.conf -echo 'vm.overcommit_memory = 1' >> /etc/sysctl.conf -echo 'fs.inotify.max_user_instances = 524288' >> /etc/sysctl.conf -echo 'fs.inotify.max_user_watches = 10240001' >> /etc/sysctl.conf -echo 'fs.pipe-max-size = 4194304' >> /etc/sysctl.conf -echo 'fs.aio-max-nr = 262144' >> /etc/sysctl.conf -echo 'kernel.pid_max = 65535' >> /etc/sysctl.conf -echo 'kernel.watchdog_thresh = 5' >> /etc/sysctl.conf -echo 'kernel.hung_task_timeout_secs = 5' >> /etc/sysctl.conf - -#See https://help.aliyun.com/document_detail/118806.html#uicontrol-e50-ddj-w0y -sed -r -i "s@#{0,}?net.ipv4.tcp_tw_recycle ?= ?(0|1|2)@net.ipv4.tcp_tw_recycle = 0@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.tcp_tw_reuse ?= ?(0|1)@net.ipv4.tcp_tw_reuse = 0@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.conf.all.rp_filter ?= ?(0|1|2)@net.ipv4.conf.all.rp_filter = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.conf.default.rp_filter ?= ?(0|1|2)@net.ipv4.conf.default.rp_filter = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.ip_forward ?= ?(0|1)@net.ipv4.ip_forward = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-arptables ?= ?(0|1)@net.bridge.bridge-nf-call-arptables = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-ip6tables ?= ?(0|1)@net.bridge.bridge-nf-call-ip6tables = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-iptables ?= ?(0|1)@net.bridge.bridge-nf-call-iptables = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.ip_local_reserved_ports ?= ?([0-9]{1,}-{0,1},{0,1}){1,}@net.ipv4.ip_local_reserved_ports = 30000-32767@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?vm.max_map_count ?= ?([0-9]{1,})@vm.max_map_count = 262144@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?vm.swappiness ?= ?([0-9]{1,})@vm.swappiness = 0@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?fs.inotify.max_user_instances ?= ?([0-9]{1,})@fs.inotify.max_user_instances = 524288@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?kernel.pid_max ?= ?([0-9]{1,})@kernel.pid_max = 65535@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?vm.overcommit_memory ?= ?(0|1|2)@vm.overcommit_memory = 0@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?fs.inotify.max_user_watches ?= ?([0-9]{1,})@fs.inotify.max_user_watches = 524288@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?fs.pipe-max-size ?= ?([0-9]{1,})@fs.pipe-max-size = 4194304@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.core.netdev_max_backlog ?= ?([0-9]{1,})@net.core.netdev_max_backlog = 65535@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.core.rmem_max ?= ?([0-9]{1,})@net.core.rmem_max = 33554432@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.core.wmem_max ?= ?([0-9]{1,})@net.core.wmem_max = 33554432@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.tcp_max_syn_backlog ?= ?([0-9]{1,})@net.ipv4.tcp_max_syn_backlog = 1048576@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.neigh.default.gc_thresh1 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh1 = 512@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.neigh.default.gc_thresh2 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh2 = 2048@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.neigh.default.gc_thresh3 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh3 = 4096@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.core.somaxconn ?= ?([0-9]{1,})@net.core.somaxconn = 32768@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.conf.eth0.arp_accept ?= ?(0|1)@net.ipv4.conf.eth0.arp_accept = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?fs.aio-max-nr ?= ?([0-9]{1,})@fs.aio-max-nr = 262144@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.tcp_retries2 ?= ?([0-9]{1,})@net.ipv4.tcp_retries2 = 15@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.tcp_max_tw_buckets ?= ?([0-9]{1,})@net.ipv4.tcp_max_tw_buckets = 1048576@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.tcp_max_orphans ?= ?([0-9]{1,})@net.ipv4.tcp_max_orphans = 65535@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.udp_rmem_min ?= ?([0-9]{1,})@net.ipv4.udp_rmem_min = 131072@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.udp_wmem_min ?= ?([0-9]{1,})@net.ipv4.udp_wmem_min = 131072@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.conf.all.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.all.arp_ignore = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.conf.default.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.default.arp_ignore = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?kernel.watchdog_thresh ?= ?([0-9]{1,})@kernel.watchdog_thresh = 5@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?kernel.hung_task_timeout_secs ?= ?([0-9]{1,})@kernel.hung_task_timeout_secs = 5@g" /etc/sysctl.conf - -tmpfile="$$.tmp" -awk ' !x[$0]++{print > "'$tmpfile'"}' /etc/sysctl.conf -mv $tmpfile /etc/sysctl.conf - -# ulimit -echo "* soft nofile 1048576" >> /etc/security/limits.conf -echo "* hard nofile 1048576" >> /etc/security/limits.conf -echo "* soft nproc 65536" >> /etc/security/limits.conf -echo "* hard nproc 65536" >> /etc/security/limits.conf -echo "* soft memlock unlimited" >> /etc/security/limits.conf -echo "* hard memlock unlimited" >> /etc/security/limits.conf - -sed -r -i "s@#{0,}?\* soft nofile ?([0-9]{1,})@\* soft nofile 1048576@g" /etc/security/limits.conf -sed -r -i "s@#{0,}?\* hard nofile ?([0-9]{1,})@\* hard nofile 1048576@g" /etc/security/limits.conf -sed -r -i "s@#{0,}?\* soft nproc ?([0-9]{1,})@\* soft nproc 65536@g" /etc/security/limits.conf -sed -r -i "s@#{0,}?\* hard nproc ?([0-9]{1,})@\* hard nproc 65536@g" /etc/security/limits.conf -sed -r -i "s@#{0,}?\* soft memlock ?([0-9]{1,}([TGKM]B){0,1}|unlimited)@\* soft memlock unlimited@g" /etc/security/limits.conf -sed -r -i "s@#{0,}?\* hard memlock ?([0-9]{1,}([TGKM]B){0,1}|unlimited)@\* hard memlock unlimited@g" /etc/security/limits.conf - -tmpfile="$$.tmp" -awk ' !x[$0]++{print > "'$tmpfile'"}' /etc/security/limits.conf -mv $tmpfile /etc/security/limits.conf - -systemctl stop firewalld 1>/dev/null 2>/dev/null -systemctl disable firewalld 1>/dev/null 2>/dev/null -systemctl stop ufw 1>/dev/null 2>/dev/null -systemctl disable ufw 1>/dev/null 2>/dev/null - -modinfo br_netfilter > /dev/null 2>&1 -if [ $? -eq 0 ]; then - modprobe br_netfilter - mkdir -p /etc/modules-load.d - echo 'br_netfilter' > /etc/modules-load.d/kubekey-br_netfilter.conf -fi - -modinfo overlay > /dev/null 2>&1 -if [ $? -eq 0 ]; then - modprobe overlay - echo 'overlay' >> /etc/modules-load.d/kubekey-br_netfilter.conf -fi - -modprobe ip_vs -modprobe ip_vs_rr -modprobe ip_vs_wrr -modprobe ip_vs_sh - -cat > /etc/modules-load.d/kube_proxy-ipvs.conf << EOF -ip_vs -ip_vs_rr -ip_vs_wrr -ip_vs_sh -EOF - -modprobe nf_conntrack_ipv4 1>/dev/null 2>/dev/null -if [ $? -eq 0 ]; then - echo 'nf_conntrack_ipv4' > /etc/modules-load.d/kube_proxy-ipvs.conf -else - modprobe nf_conntrack - echo 'nf_conntrack' > /etc/modules-load.d/kube_proxy-ipvs.conf -fi -sysctl -p - -sed -i ':a;$!{N;ba};s@# kubekey hosts BEGIN.*# kubekey hosts END@@' /etc/hosts -sed -i '/^$/N;/\n$/N;//D' /etc/hosts - -cat >>/etc/hosts< /proc/sys/vm/drop_caches - -# Make sure the iptables utility doesn't use the nftables backend. -update-alternatives --set iptables /usr/sbin/iptables-legacy >/dev/null 2>&1 || true -update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy >/dev/null 2>&1 || true -update-alternatives --set arptables /usr/sbin/arptables-legacy >/dev/null 2>&1 || true -update-alternatives --set ebtables /usr/sbin/ebtables-legacy >/dev/null 2>&1 || true - - `))) - -func GenerateHosts(runtime connector.ModuleRuntime, kubeConf *common.KubeConf) []string { - var lbHost string - var hostsList []string - - if kubeConf.Cluster.ControlPlaneEndpoint.Address != "" { - lbHost = fmt.Sprintf("%s %s", kubeConf.Cluster.ControlPlaneEndpoint.Address, kubeConf.Cluster.ControlPlaneEndpoint.Domain) - } - - for _, host := range runtime.GetAllHosts() { - if host.GetName() != "" { - hostsList = append(hostsList, fmt.Sprintf("%s %s.%s %s", - host.GetInternalAddress(), - host.GetName(), - kubeConf.Cluster.Kubernetes.ClusterName, - host.GetName())) - } - } - - if len(runtime.GetHostsByRole(common.Registry)) > 0 { - if kubeConf.Cluster.Registry.PrivateRegistry != "" { - hostsList = append(hostsList, fmt.Sprintf("%s %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalAddress(), kubeConf.Cluster.Registry.PrivateRegistry)) - } else { - hostsList = append(hostsList, fmt.Sprintf("%s %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalAddress(), registry.RegistryCertificateBaseName)) - } - - } - - hostsList = append(hostsList, lbHost) - return hostsList -} diff --git a/cmd/kk/pkg/bootstrap/precheck/const.go b/cmd/kk/pkg/bootstrap/precheck/const.go deleted file mode 100644 index 71d5db20..00000000 --- a/cmd/kk/pkg/bootstrap/precheck/const.go +++ /dev/null @@ -1,59 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package precheck - -const ( - // command software - sudo = "sudo" - curl = "curl" - openssl = "openssl" - ebtables = "ebtables" - socat = "socat" - ipset = "ipset" - ipvsadm = "ipvsadm" - conntrack = "conntrack" - chrony = "chronyd" - docker = "docker" - containerd = "containerd" - showmount = "showmount" - rbd = "rbd" - glusterfs = "glusterfs" - - // extra command tools - nfs = "nfs" - ceph = "ceph" - - UnknownVersion = "UnknownVersion" -) - -// defines the base software to be checked. -var baseSoftware = []string{ - sudo, - curl, - openssl, - ebtables, - socat, - ipset, - ipvsadm, - conntrack, - chrony, - docker, - containerd, - showmount, - rbd, - glusterfs, -} diff --git a/cmd/kk/pkg/bootstrap/precheck/module.go b/cmd/kk/pkg/bootstrap/precheck/module.go deleted file mode 100644 index df6583cd..00000000 --- a/cmd/kk/pkg/bootstrap/precheck/module.go +++ /dev/null @@ -1,171 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package precheck - -import ( - "time" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type GreetingsModule struct { - module.BaseTaskModule -} - -func (h *GreetingsModule) Init() { - h.Name = "GreetingsModule" - h.Desc = "Greetings" - - var timeout int64 - for _, v := range h.Runtime.GetAllHosts() { - timeout += v.GetTimeout() - } - - hello := &task.RemoteTask{ - Name: "Greetings", - Desc: "Greetings", - Hosts: h.Runtime.GetAllHosts(), - Action: new(GreetingsTask), - Parallel: true, - Timeout: time.Duration(timeout) * time.Second, - } - - h.Tasks = []task.Interface{ - hello, - } -} - -type NodePreCheckModule struct { - common.KubeModule - Skip bool -} - -func (n *NodePreCheckModule) IsSkip() bool { - return n.Skip -} - -func (n *NodePreCheckModule) Init() { - n.Name = "NodePreCheckModule" - n.Desc = "Do pre-check on cluster nodes" - - preCheck := &task.RemoteTask{ - Name: "NodePreCheck", - Desc: "A pre-check on nodes", - Hosts: n.Runtime.GetAllHosts(), - //Prepare: &prepare.FastPrepare{ - // Inject: func(runtime connector.Runtime) (bool, error) { - // if len(n.Runtime.GetHostsByRole(common.ETCD))%2 == 0 { - // logger.Log.Error("The number of etcd is even. Please configure it to be odd.") - // return false, errors.New("the number of etcd is even") - // } - // return true, nil - // }}, - Action: new(NodePreCheck), - Parallel: true, - } - - n.Tasks = []task.Interface{ - preCheck, - } -} - -type ClusterPreCheckModule struct { - common.KubeModule -} - -func (c *ClusterPreCheckModule) Init() { - c.Name = "ClusterPreCheckModule" - c.Desc = "Do pre-check on cluster" - - getKubeConfig := &task.RemoteTask{ - Name: "GetKubeConfig", - Desc: "Get KubeConfig file", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(GetKubeConfig), - Parallel: true, - } - - getAllNodesK8sVersion := &task.RemoteTask{ - Name: "GetAllNodesK8sVersion", - Desc: "Get all nodes Kubernetes version", - Hosts: c.Runtime.GetHostsByRole(common.K8s), - Action: new(GetAllNodesK8sVersion), - Parallel: true, - } - - calculateMinK8sVersion := &task.RemoteTask{ - Name: "CalculateMinK8sVersion", - Desc: "Calculate min Kubernetes version", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(CalculateMinK8sVersion), - Parallel: true, - } - - checkDesiredK8sVersion := &task.RemoteTask{ - Name: "CheckDesiredK8sVersion", - Desc: "Check desired Kubernetes version", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(CheckDesiredK8sVersion), - Parallel: true, - } - - ksVersionCheck := &task.RemoteTask{ - Name: "KsVersionCheck", - Desc: "Check KubeSphere version", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(KsVersionCheck), - Parallel: true, - } - - dependencyCheck := &task.RemoteTask{ - Name: "DependencyCheck", - Desc: "Check dependency matrix for KubeSphere and Kubernetes", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(KubeSphereExist), - }, - Action: new(DependencyCheck), - Parallel: true, - } - - getKubernetesNodesStatus := &task.RemoteTask{ - Name: "GetKubernetesNodesStatus", - Desc: "Get kubernetes nodes status", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(GetKubernetesNodesStatus), - Parallel: true, - } - - c.Tasks = []task.Interface{ - getKubeConfig, - getAllNodesK8sVersion, - calculateMinK8sVersion, - checkDesiredK8sVersion, - ksVersionCheck, - dependencyCheck, - getKubernetesNodesStatus, - } -} diff --git a/cmd/kk/pkg/bootstrap/precheck/prepares.go b/cmd/kk/pkg/bootstrap/precheck/prepares.go deleted file mode 100644 index 53c2e10a..00000000 --- a/cmd/kk/pkg/bootstrap/precheck/prepares.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package precheck - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type KubeSphereExist struct { - common.KubePrepare -} - -func (k *KubeSphereExist) PreCheck(runtime connector.Runtime) (bool, error) { - currentKsVersion, ok := k.PipelineCache.GetMustString(common.KubeSphereVersion) - if !ok { - return false, errors.New("get current KubeSphere version failed by pipeline cache") - } - if currentKsVersion != "" { - return true, nil - } - return false, nil -} diff --git a/cmd/kk/pkg/bootstrap/precheck/tasks.go b/cmd/kk/pkg/bootstrap/precheck/tasks.go deleted file mode 100644 index c6d60cb7..00000000 --- a/cmd/kk/pkg/bootstrap/precheck/tasks.go +++ /dev/null @@ -1,336 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package precheck - -import ( - "fmt" - "regexp" - "strings" - - "github.com/pkg/errors" - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -type GreetingsTask struct { - action.BaseAction -} - -func (h *GreetingsTask) Execute(runtime connector.Runtime) error { - hello, err := runtime.GetRunner().SudoCmd("echo 'Greetings, KubeKey!'", false) - if err != nil { - return err - } - logger.Log.Messagef(runtime.RemoteHost().GetName(), hello) - return nil -} - -type NodePreCheck struct { - common.KubeAction -} - -func (n *NodePreCheck) Execute(runtime connector.Runtime) error { - var results = make(map[string]string) - results["name"] = runtime.RemoteHost().GetName() - for _, software := range baseSoftware { - var ( - cmd string - ) - - switch software { - case docker: - cmd = "docker version --format '{{.Server.Version}}'" - case containerd: - cmd = "containerd --version | cut -d ' ' -f 3" - default: - cmd = fmt.Sprintf("which %s", software) - } - - switch software { - case sudo: - // sudo skip sudo prefix - default: - cmd = connector.SudoPrefix(cmd) - } - - res, err := runtime.GetRunner().Cmd(cmd, false) - switch software { - case showmount: - software = nfs - case rbd: - software = ceph - case glusterfs: - software = glusterfs - } - if err != nil || strings.Contains(res, "not found") { - results[software] = "" - } else { - // software in path - if strings.Contains(res, "bin/") { - results[software] = "y" - } else { - // get software version, e.g. docker, containerd, etc. - results[software] = res - } - } - } - - output, err := runtime.GetRunner().Cmd("date +\"%Z %H:%M:%S\"", false) - if err != nil { - results["time"] = "" - } else { - results["time"] = strings.TrimSpace(output) - } - - host := runtime.RemoteHost() - if res, ok := host.GetCache().Get(common.NodePreCheck); ok { - m := res.(map[string]string) - m = results - host.GetCache().Set(common.NodePreCheck, m) - } else { - host.GetCache().Set(common.NodePreCheck, results) - } - return nil -} - -type GetKubeConfig struct { - common.KubeAction -} - -func (g *GetKubeConfig) Execute(runtime connector.Runtime) error { - if exist, err := runtime.GetRunner().FileExist("$HOME/.kube/config"); err != nil { - return err - } else { - if exist { - return nil - } else { - if exist, err := runtime.GetRunner().FileExist("/etc/kubernetes/admin.conf"); err != nil { - return err - } else { - if exist { - if _, err := runtime.GetRunner().Cmd("mkdir -p $HOME/.kube", false); err != nil { - return err - } - if _, err := runtime.GetRunner().SudoCmd("cp /etc/kubernetes/admin.conf $HOME/.kube/config", false); err != nil { - return err - } - userId, err := runtime.GetRunner().Cmd("echo $(id -u)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user id failed") - } - - userGroupId, err := runtime.GetRunner().Cmd("echo $(id -g)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user group id failed") - } - - chownKubeConfig := fmt.Sprintf("chown -R %s:%s $HOME/.kube", userId, userGroupId) - if _, err := runtime.GetRunner().SudoCmd(chownKubeConfig, false); err != nil { - return errors.Wrap(errors.WithStack(err), "chown user kube config failed") - } - } - } - } - } - return errors.New("kube config not found") -} - -type GetAllNodesK8sVersion struct { - common.KubeAction -} - -func (g *GetAllNodesK8sVersion) Execute(runtime connector.Runtime) error { - var nodeK8sVersion string - kubeletVersionInfo, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubelet --version", false) - if err != nil { - return errors.Wrap(err, "get current kubelet version failed") - } - nodeK8sVersion = strings.Split(kubeletVersionInfo, " ")[1] - - host := runtime.RemoteHost() - host.GetCache().Set(common.NodeK8sVersion, nodeK8sVersion) - - if host.IsRole(common.Master) { - apiserverVersion, err := runtime.GetRunner().SudoCmd( - "cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep 'image:' | rev | cut -d ':' -f1 | rev", - false) - if err != nil { - return errors.Wrap(err, "get current kube-apiserver version failed") - } - - apiserverSemanticVersion, err := versionutil.ParseSemantic(apiserverVersion) - if err != nil { - return errors.Wrap(err, "parse kube-apiserver version failed") - } - - kubeletSemanticVersion, err := versionutil.ParseSemantic(nodeK8sVersion) - if err != nil { - return errors.Wrap(err, "parse kubelet version failed") - } - - if apiserverSemanticVersion.LessThan(kubeletSemanticVersion) { - host.GetCache().Set(common.NodeK8sVersion, apiserverVersion) - } - } - - return nil -} - -type CalculateMinK8sVersion struct { - common.KubeAction -} - -func (g *CalculateMinK8sVersion) Execute(runtime connector.Runtime) error { - versionList := make([]*versionutil.Version, 0, len(runtime.GetHostsByRole(common.K8s))) - for _, host := range runtime.GetHostsByRole(common.K8s) { - version, ok := host.GetCache().GetMustString(common.NodeK8sVersion) - if !ok { - return errors.Errorf("get node %s Kubernetes version failed by host cache", host.GetName()) - } - if versionObj, err := versionutil.ParseSemantic(version); err != nil { - return errors.Wrap(err, "parse node version failed") - } else { - versionList = append(versionList, versionObj) - } - } - - minVersion := versionList[0] - for _, version := range versionList { - if !minVersion.LessThan(version) { - minVersion = version - } - } - g.PipelineCache.Set(common.K8sVersion, fmt.Sprintf("v%s", minVersion)) - return nil -} - -type CheckDesiredK8sVersion struct { - common.KubeAction -} - -func (k *CheckDesiredK8sVersion) Execute(_ connector.Runtime) error { - if ok := kubernetes.VersionSupport(k.KubeConf.Cluster.Kubernetes.Version); !ok { - return errors.New(fmt.Sprintf("does not support upgrade to Kubernetes %s", - k.KubeConf.Cluster.Kubernetes.Version)) - } - k.PipelineCache.Set(common.DesiredK8sVersion, k.KubeConf.Cluster.Kubernetes.Version) - return nil -} - -type KsVersionCheck struct { - common.KubeAction -} - -func (k *KsVersionCheck) Execute(runtime connector.Runtime) error { - ksVersionStr, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl get deploy -n kubesphere-system ks-console -o jsonpath='{.metadata.labels.version}'", - false) - if err != nil { - if k.KubeConf.Cluster.KubeSphere.Enabled { - return errors.Wrap(err, "get kubeSphere version failed") - } else { - ksVersionStr = "" - } - } - - ccKsVersionStr, ccErr := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl get ClusterConfiguration ks-installer -n kubesphere-system -o jsonpath='{.metadata.labels.version}'", - false) - - if ccErr == nil && versionutil.MustParseSemantic(ccKsVersionStr).AtLeast(versionutil.MustParseSemantic("v3.1.0")) { - ksVersionStr = ccKsVersionStr - } - - k.PipelineCache.Set(common.KubeSphereVersion, ksVersionStr) - return nil -} - -type DependencyCheck struct { - common.KubeAction -} - -func (d *DependencyCheck) Execute(_ connector.Runtime) error { - currentKsVersion, ok := d.PipelineCache.GetMustString(common.KubeSphereVersion) - if !ok { - return errors.New("get current KubeSphere version failed by pipeline cache") - } - desiredVersion := d.KubeConf.Cluster.KubeSphere.Version - - if d.KubeConf.Cluster.KubeSphere.Enabled { - var version string - if latest, ok := kubesphere.LatestRelease(desiredVersion); ok { - version = latest.Version - } else if ks, ok := kubesphere.DevRelease(desiredVersion); ok { - version = ks.Version - } else { - r := regexp.MustCompile("v(\\d+\\.)?(\\d+\\.)?(\\*|\\d+)") - version = r.FindString(desiredVersion) - } - - ksInstaller, ok := kubesphere.VersionMap[version] - if !ok { - return errors.New(fmt.Sprintf("Unsupported version: %s", desiredVersion)) - } - - if currentKsVersion != desiredVersion { - if ok := ksInstaller.UpgradeSupport(currentKsVersion); !ok { - return errors.New(fmt.Sprintf("Unsupported upgrade plan: %s to %s", currentKsVersion, desiredVersion)) - } - } - - if ok := ksInstaller.K8sSupport(d.KubeConf.Cluster.Kubernetes.Version); !ok { - return errors.New(fmt.Sprintf("KubeSphere %s does not support running on Kubernetes %s", - version, d.KubeConf.Cluster.Kubernetes.Version)) - } - } else { - ksInstaller, ok := kubesphere.VersionMap[currentKsVersion] - if !ok { - return errors.New(fmt.Sprintf("Unsupported version: %s", currentKsVersion)) - } - - if ok := ksInstaller.K8sSupport(d.KubeConf.Cluster.Kubernetes.Version); !ok { - return errors.New(fmt.Sprintf("KubeSphere %s does not support running on Kubernetes %s", - currentKsVersion, d.KubeConf.Cluster.Kubernetes.Version)) - } - } - return nil -} - -type GetKubernetesNodesStatus struct { - common.KubeAction -} - -func (g *GetKubernetesNodesStatus) Execute(runtime connector.Runtime) error { - nodeStatus, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl get node -o wide", false) - if err != nil { - return err - } - g.PipelineCache.Set(common.ClusterNodeStatus, nodeStatus) - - cri, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl get node -o jsonpath=\"{.items[*].status.nodeInfo.containerRuntimeVersion}\"", false) - if err != nil { - return err - } - g.PipelineCache.Set(common.ClusterNodeCRIRuntimes, cri) - return nil -} diff --git a/cmd/kk/pkg/bootstrap/registry/certs.go b/cmd/kk/pkg/bootstrap/registry/certs.go deleted file mode 100644 index 8b591847..00000000 --- a/cmd/kk/pkg/bootstrap/registry/certs.go +++ /dev/null @@ -1,146 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package registry - -import ( - "crypto/x509" - "fmt" - "net" - "path/filepath" - "strings" - - "github.com/pkg/errors" - "k8s.io/client-go/util/cert" - certutil "k8s.io/client-go/util/cert" - netutils "k8s.io/utils/net" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils/certs" -) - -const ( - RegistryCertificateBaseName = "dockerhub.kubekey.local" - LocalCertsDir = "localCertsDir" - CertsFileList = "certsFileList" -) - -// KubekeyCertRegistryCA is the definition of the root CA used by the hosted etcd server. -func KubekeyCertRegistryCA() *certs.KubekeyCert { - return &certs.KubekeyCert{ - Name: "registry-ca", - LongName: "self-signed CA to provision identities for registry", - BaseName: "ca", - Config: certs.CertConfig{ - Config: certutil.Config{ - CommonName: "registry-ca", - }, - }, - } -} - -// KubekeyCertRegistryServer is the definition of the cert for etcd admin. -func KubekeyCertRegistryServer(baseName string, altNames *certutil.AltNames) *certs.KubekeyCert { - return &certs.KubekeyCert{ - Name: "registry-server", - LongName: "certificate for registry server", - BaseName: baseName, - CAName: "registry-ca", - Config: certs.CertConfig{ - Config: certutil.Config{ - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - AltNames: *altNames, - CommonName: baseName, - }, - }, - } -} - -type FetchCerts struct { - common.KubeAction -} - -func (f *FetchCerts) Execute(runtime connector.Runtime) error { - src := "/etc/ssl/registry/ssl" - dst := fmt.Sprintf("%s/pki/registry", runtime.GetWorkDir()) - - certs, err := runtime.GetRunner().SudoCmd("ls /etc/ssl/registry/ssl/ | grep .pem", false) - if err != nil { - return nil - } - - certsList := strings.Split(certs, "\r\n") - if len(certsList) > 0 { - for _, cert := range certsList { - if err := runtime.GetRunner().Fetch(filepath.Join(dst, cert), filepath.Join(src, cert)); err != nil { - return errors.Wrap(err, fmt.Sprintf("Fetch %s failed", filepath.Join(src, cert))) - } - } - } - - return nil -} - -type GenerateCerts struct { - common.KubeAction -} - -func (g *GenerateCerts) Execute(runtime connector.Runtime) error { - - pkiPath := fmt.Sprintf("%s/pki/registry", runtime.GetWorkDir()) - - var altName cert.AltNames - - dnsList := []string{"localhost", RegistryCertificateBaseName} - ipList := []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback} - - for _, h := range runtime.GetHostsByRole(common.Registry) { - dnsList = append(dnsList, h.GetName()) - ipList = append(ipList, netutils.ParseIPSloppy(h.GetInternalAddress())) - } - altName.DNSNames = dnsList - altName.IPs = ipList - - files := []string{"ca.pem", "ca-key.pem", fmt.Sprintf("%s.pem", g.KubeConf.Cluster.Registry.PrivateRegistry), fmt.Sprintf("%s-key.pem", g.KubeConf.Cluster.Registry.PrivateRegistry)} - - // CA - certsList := []*certs.KubekeyCert{KubekeyCertRegistryCA()} - - // Certs - certsList = append(certsList, KubekeyCertRegistryServer(g.KubeConf.Cluster.Registry.PrivateRegistry, &altName)) - - var lastCACert *certs.KubekeyCert - for _, c := range certsList { - if c.CAName == "" { - err := certs.GenerateCA(c, pkiPath, g.KubeConf) - if err != nil { - return err - } - lastCACert = c - } else { - err := certs.GenerateCerts(c, lastCACert, pkiPath, g.KubeConf) - if err != nil { - return err - } - } - } - - g.ModuleCache.Set(LocalCertsDir, pkiPath) - g.ModuleCache.Set(CertsFileList, files) - - return nil -} diff --git a/cmd/kk/pkg/bootstrap/registry/module.go b/cmd/kk/pkg/bootstrap/registry/module.go deleted file mode 100644 index 284b88ba..00000000 --- a/cmd/kk/pkg/bootstrap/registry/module.go +++ /dev/null @@ -1,281 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package registry - -import ( - "fmt" - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container" - docker_template "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type RegistryCertsModule struct { - common.KubeModule - Skip bool -} - -func (p *RegistryCertsModule) IsSkip() bool { - return p.Skip -} - -func (i *RegistryCertsModule) Init() { - i.Name = "InitRegistryModule" - i.Desc = "Init a local registry" - - fetchCerts := &task.RemoteTask{ - Name: "FetchRegistryCerts", - Desc: "Fetch registry certs", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Prepare: new(FirstRegistryNode), - Action: new(FetchCerts), - Parallel: false, - } - - generateCerts := &task.LocalTask{ - Name: "GenerateRegistryCerts", - Desc: "Generate registry Certs", - Action: new(GenerateCerts), - } - - syncCertsFile := &task.RemoteTask{ - Name: "SyncCertsFile", - Desc: "Synchronize certs file", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: new(SyncCertsFile), - Parallel: true, - Retry: 1, - } - - syncCertsToAllNodes := &task.RemoteTask{ - Name: "SyncCertsFileToAllNodes", - Desc: "Synchronize certs file to all nodes", - Hosts: i.Runtime.GetAllHosts(), - Action: new(SyncCertsToAllNodes), - Parallel: true, - Retry: 1, - } - - i.Tasks = []task.Interface{ - fetchCerts, - generateCerts, - syncCertsFile, - syncCertsToAllNodes, - } - -} - -type InstallRegistryModule struct { - common.KubeModule -} - -func (i *InstallRegistryModule) Init() { - i.Name = "InstallRegistryModule" - i.Desc = "Install local registry" - - switch i.KubeConf.Cluster.Registry.Type { - case common.Harbor: - i.Tasks = InstallHarbor(i) - default: - i.Tasks = InstallRegistry(i) - } -} - -func InstallRegistry(i *InstallRegistryModule) []task.Interface { - installRegistryBinary := &task.RemoteTask{ - Name: "InstallRegistryBinary", - Desc: "Install registry binary", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: new(InstallRegistryBinary), - Parallel: true, - Retry: 1, - } - - generateRegistryService := &task.RemoteTask{ - Name: "GenerateRegistryService", - Desc: "Generate registry service", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: &action.Template{ - Template: templates.RegistryServiceTempl, - Dst: "/etc/systemd/system/registry.service", - }, - Parallel: true, - Retry: 1, - } - - generateRegistryConfig := &task.RemoteTask{ - Name: "GenerateRegistryConfig", - Desc: "Generate registry config", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: &action.Template{ - Template: templates.RegistryConfigTempl, - Dst: "/etc/kubekey/registry/config.yaml", - Data: util.Data{ - "Certificate": fmt.Sprintf("%s.pem", i.KubeConf.Cluster.Registry.PrivateRegistry), - "Key": fmt.Sprintf("%s-key.pem", i.KubeConf.Cluster.Registry.PrivateRegistry), - }, - }, - Parallel: true, - Retry: 1, - } - - startRegistryService := &task.RemoteTask{ - Name: "StartRegistryService", - Desc: "Start registry service", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: new(StartRegistryService), - Parallel: true, - Retry: 1, - } - - return []task.Interface{ - installRegistryBinary, - generateRegistryService, - generateRegistryConfig, - startRegistryService, - } -} - -func InstallHarbor(i *InstallRegistryModule) []task.Interface { - // Install docker - syncBinaries := &task.RemoteTask{ - Name: "SyncDockerBinaries", - Desc: "Sync docker binaries", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Prepare: &prepare.PrepareCollection{ - &container.DockerExist{Not: true}, - }, - Action: new(container.SyncDockerBinaries), - Parallel: true, - Retry: 2, - } - - generateDockerService := &task.RemoteTask{ - Name: "GenerateDockerService", - Desc: "Generate docker service", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Prepare: &prepare.PrepareCollection{ - &container.DockerExist{Not: true}, - }, - Action: &action.Template{ - Template: docker_template.DockerService, - Dst: filepath.Join("/etc/systemd/system", docker_template.DockerService.Name()), - }, - Parallel: true, - } - - generateDockerConfig := &task.RemoteTask{ - Name: "GenerateDockerConfig", - Desc: "Generate docker config", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Prepare: &prepare.PrepareCollection{ - &container.DockerExist{Not: true}, - }, - Action: &action.Template{ - Template: docker_template.DockerConfig, - Dst: filepath.Join("/etc/docker/", docker_template.DockerConfig.Name()), - Data: util.Data{ - "Mirrors": docker_template.Mirrors(i.KubeConf), - "InsecureRegistries": docker_template.InsecureRegistries(i.KubeConf), - }, - }, - Parallel: true, - } - - enableDocker := &task.RemoteTask{ - Name: "EnableDocker", - Desc: "Enable docker", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Prepare: &prepare.PrepareCollection{ - &container.DockerExist{Not: true}, - }, - Action: new(container.EnableDocker), - Parallel: true, - } - - // Install docker compose - installDockerCompose := &task.RemoteTask{ - Name: "InstallDockerCompose", - Desc: "Install docker compose", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: new(InstallDockerCompose), - Parallel: true, - Retry: 2, - } - - // Install Harbor - syncHarborPackage := &task.RemoteTask{ - Name: "SyncHarborPackage", - Desc: "Sync harbor package", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: new(SyncHarborPackage), - Parallel: true, - Retry: 2, - } - - // generate Harbor Systemd - generateHarborService := &task.RemoteTask{ - Name: "GenerateHarborService", - Desc: "Generate harbor service", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: &action.Template{ - Template: templates.HarborServiceTempl, - Dst: "/etc/systemd/system/harbor.service", - Data: util.Data{ - "Harbor_install_path": "/opt/harbor", - }, - }, - Parallel: true, - Retry: 1, - } - - generateHarborConfig := &task.RemoteTask{ - Name: "GenerateHarborConfig", - Desc: "Generate harbor config", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: new(GenerateHarborConfig), - Parallel: true, - Retry: 1, - } - - startHarbor := &task.RemoteTask{ - Name: "StartHarbor", - Desc: "start harbor", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: new(StartHarbor), - Parallel: true, - Retry: 2, - } - - return []task.Interface{ - syncBinaries, - generateDockerService, - generateDockerConfig, - enableDocker, - installDockerCompose, - syncHarborPackage, - generateHarborService, - generateHarborConfig, - startHarbor, - } -} diff --git a/cmd/kk/pkg/bootstrap/registry/prepares.go b/cmd/kk/pkg/bootstrap/registry/prepares.go deleted file mode 100644 index 6db26676..00000000 --- a/cmd/kk/pkg/bootstrap/registry/prepares.go +++ /dev/null @@ -1,34 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package registry - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type FirstRegistryNode struct { - common.KubePrepare - Not bool -} - -func (f *FirstRegistryNode) PreCheck(runtime connector.Runtime) (bool, error) { - if runtime.GetHostsByRole(common.Registry)[0].GetName() == runtime.RemoteHost().GetName() { - return !f.Not, nil - } - return f.Not, nil -} diff --git a/cmd/kk/pkg/bootstrap/registry/tasks.go b/cmd/kk/pkg/bootstrap/registry/tasks.go deleted file mode 100644 index e3b9e62a..00000000 --- a/cmd/kk/pkg/bootstrap/registry/tasks.go +++ /dev/null @@ -1,256 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package registry - -import ( - "fmt" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -type SyncCertsFile struct { - common.KubeAction -} - -func (s *SyncCertsFile) Execute(runtime connector.Runtime) error { - localCertsDir, ok := s.ModuleCache.Get(LocalCertsDir) - if !ok { - return errors.New("get registry local certs dir by module cache failed") - } - files, ok := s.ModuleCache.Get(CertsFileList) - if !ok { - return errors.New("get registry certs file list by module cache failed") - } - dir := localCertsDir.(string) - fileList := files.([]string) - - for _, fileName := range fileList { - if err := runtime.GetRunner().SudoScp(filepath.Join(dir, fileName), filepath.Join(common.RegistryCertDir, fileName)); err != nil { - return errors.Wrap(errors.WithStack(err), "scp registry certs file failed") - } - } - - return nil -} - -type SyncCertsToAllNodes struct { - common.KubeAction -} - -func (s *SyncCertsToAllNodes) Execute(runtime connector.Runtime) error { - localCertsDir, ok := s.ModuleCache.Get(LocalCertsDir) - if !ok { - return errors.New("get registry local certs dir by module cache failed") - } - files, ok := s.ModuleCache.Get(CertsFileList) - if !ok { - return errors.New("get registry certs file list by module cache failed") - } - dir := localCertsDir.(string) - fileList := files.([]string) - - for _, fileName := range fileList { - var dstFileName string - switch fileName { - case "ca.pem": - dstFileName = "ca.crt" - case "ca-key.pem": - continue - default: - if strings.HasSuffix(fileName, "-key.pem") { - dstFileName = strings.Replace(fileName, "-key.pem", ".key", -1) - } else { - dstFileName = strings.Replace(fileName, ".pem", ".cert", -1) - } - } - - if err := runtime.GetRunner().SudoScp(filepath.Join(dir, fileName), filepath.Join(filepath.Join("/etc/docker/certs.d", s.KubeConf.Cluster.Registry.PrivateRegistry), dstFileName)); err != nil { - return errors.Wrap(errors.WithStack(err), "scp registry certs file to /etc/docker/certs.d/ failed") - } - - if err := runtime.GetRunner().SudoScp(filepath.Join(dir, fileName), filepath.Join(common.RegistryCertDir, dstFileName)); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("scp registry certs file to %s failed", common.RegistryCertDir)) - } - } - - return nil -} - -type InstallRegistryBinary struct { - common.KubeAction -} - -func (g *InstallRegistryBinary) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binariesMapObj, ok := g.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - registry, ok := binariesMap[common.Registry] - if !ok { - return errors.New("get KubeBinary key registry by pipeline cache failed") - } - - dst := filepath.Join(common.TmpDir, registry.FileName) - if err := runtime.GetRunner().Scp(registry.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), "sync etcd tar.gz failed") - } - - installCmd := fmt.Sprintf("tar -zxf %s && mv -f registry /usr/local/bin/ && chmod +x /usr/local/bin/registry", dst) - if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "install etcd binaries failed") - } - return nil -} - -type StartRegistryService struct { - common.KubeAction -} - -func (g *StartRegistryService) Execute(runtime connector.Runtime) error { - installCmd := "systemctl daemon-reload && systemctl enable registry && systemctl restart registry" - if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "start registry service failed") - } - - fmt.Println() - fmt.Println(fmt.Sprintf("Local image registry created successfully. Address: %s", g.KubeConf.Cluster.Registry.PrivateRegistry)) - fmt.Println() - - return nil -} - -type InstallDockerCompose struct { - common.KubeAction -} - -func (g *InstallDockerCompose) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binariesMapObj, ok := g.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - compose, ok := binariesMap[common.DockerCompose] - if !ok { - return errors.New("get KubeBinary key docker-compose by pipeline cache failed") - } - - dst := filepath.Join(common.TmpDir, compose.FileName) - if err := runtime.GetRunner().Scp(compose.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), "sync docker-compose failed") - } - - installCmd := fmt.Sprintf("mv -f %s /usr/local/bin/docker-compose && chmod +x /usr/local/bin/docker-compose", dst) - if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "install dokcer-compose failed") - } - - return nil -} - -type SyncHarborPackage struct { - common.KubeAction -} - -func (g *SyncHarborPackage) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binariesMapObj, ok := g.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - harbor, ok := binariesMap[common.Harbor] - if !ok { - return errors.New("get KubeBinary key harbor by pipeline cache failed") - } - - dst := filepath.Join(common.TmpDir, harbor.FileName) - if err := runtime.GetRunner().Scp(harbor.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), "sync harbor package failed") - } - - installCmd := fmt.Sprintf("tar -zxvf %s -C /opt", dst) - if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "unzip harbor package failed") - } - - return nil -} - -type GenerateHarborConfig struct { - common.KubeAction -} - -func (g *GenerateHarborConfig) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - templateAction := action.Template{ - Template: templates.HarborConfigTempl, - Dst: "/opt/harbor/harbor.yml", - Data: util.Data{ - "Domain": host.GetName(), - "Certificate": fmt.Sprintf("%s.pem", RegistryCertificateBaseName), - "Key": fmt.Sprintf("%s-key.pem", RegistryCertificateBaseName), - "Password": templates.Password(g.KubeConf, RegistryCertificateBaseName), - }, - } - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type StartHarbor struct { - common.KubeAction -} - -func (g *StartHarbor) Execute(runtime connector.Runtime) error { - startCmd := "cd /opt/harbor && chmod +x install.sh && export PATH=$PATH:/usr/local/bin; ./install.sh --with-notary --with-trivy --with-chartmuseum && systemctl daemon-reload && systemctl enable harbor && systemctl restart harbor" - if _, err := runtime.GetRunner().SudoCmd(startCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "start harbor failed") - } - - fmt.Println() - fmt.Println(fmt.Sprintf("Local image registry created successfully. Address: %s", g.KubeConf.Cluster.Registry.PrivateRegistry)) - fmt.Println() - - return nil -} diff --git a/cmd/kk/pkg/bootstrap/registry/templates/harbor.go b/cmd/kk/pkg/bootstrap/registry/templates/harbor.go deleted file mode 100644 index 7e97a46d..00000000 --- a/cmd/kk/pkg/bootstrap/registry/templates/harbor.go +++ /dev/null @@ -1,149 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package templates - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry" - "strings" - "text/template" - - "github.com/lithammer/dedent" -) - -var ( - // HarborServiceTempl defines the template of registry's configuration file. - HarborServiceTempl = template.Must(template.New("harborSerivce").Parse( - dedent.Dedent(`[Unit] -Description=Harbor -After=docker.service systemd-networkd.service systemd-resolved.service -Requires=docker.service - -[Service] -Type=simple -ExecStart=/usr/local/bin/docker-compose -f {{ .Harbor_install_path }}/docker-compose.yml up -ExecStop=/usr/local/bin/docker-compose -f {{ .Harbor_install_path }}/docker-compose.yml down -Restart=on-failure -[Install] -WantedBy=multi-user.target - `))) - // HarborConfigTempl defines the template of registry's configuration file. - HarborConfigTempl = template.Must(template.New("harborConfig").Parse( - dedent.Dedent(`# Configuration file of Harbor - -# The IP address or hostname to access admin UI and registry service. -# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. -hostname: {{ .Domain }} - -# http related config -http: - # port for http, default is 80. If https enabled, this port will redirect to https port - port: 80 - -# https related config -https: - # https port for harbor, default is 443 - port: 443 - # The path of cert and key files for nginx - certificate: /etc/ssl/registry/ssl/{{ .Certificate }} - private_key: /etc/ssl/registry/ssl/{{ .Key }} - -# The initial password of Harbor admin -# It only works in first time to install harbor -# Remember Change the admin password from UI after launching Harbor. -harbor_admin_password: Harbor12345 - -# Harbor DB configuration -database: - # The password for the root user of Harbor DB. Change this before any production use. - password: root123 - # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained. - max_idle_conns: 100 - # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections. - # Note: the default number of connections is 1024 for postgres of harbor. - max_open_conns: 900 - -# The default data volume -data_volume: /mnt/registry - -# Trivy configuration -# -# Trivy DB contains vulnerability information from NVD, Red Hat, and many other upstream vulnerability databases. -# It is downloaded by Trivy from the GitHub release page https://github.com/aquasecurity/trivy-db/releases and cached -# in the local file system. In addition, the database contains the update timestamp so Trivy can detect whether it -# should download a newer version from the Internet or use the cached one. Currently, the database is updated every -# 12 hours and published as a new release to GitHub. -trivy: - # ignoreUnfixed The flag to display only fixed vulnerabilities - ignore_unfixed: false - # skipUpdate The flag to enable or disable Trivy DB downloads from GitHub - # - skip_update: false - # - # insecure The flag to skip verifying registry certificate - insecure: false - -jobservice: - # Maximum number of job workers in job service - max_job_workers: 10 - -notification: - # Maximum retry count for webhook job - webhook_job_max_retry: 10 - -chart: - # Change the value of absolute_url to enabled can enable absolute url in chart - absolute_url: disabled - -# Log configurations -log: - # options are debug, info, warning, error, fatal - level: info - # configs for logs in local storage - local: - # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. - rotate_count: 50 - # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. - # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G - # are all valid. - rotate_size: 200M - # The directory on your host that store log - location: /var/log/harbor - -#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! -_version: 2.4.0 - -# Global proxy -proxy: - http_proxy: - https_proxy: - no_proxy: - components: - - core - - jobservice - - trivy - - `))) -) - -func Password(kubeConf *common.KubeConf, domain string) string { - auths := registry.DockerRegistryAuthEntries(kubeConf.Cluster.Registry.Auths) - for repo, entry := range auths { - if strings.Contains(repo, domain) { - return entry.Password - } - } - - return "Harbor12345" -} \ No newline at end of file diff --git a/cmd/kk/pkg/bootstrap/registry/templates/registry.go b/cmd/kk/pkg/bootstrap/registry/templates/registry.go deleted file mode 100644 index 527935f1..00000000 --- a/cmd/kk/pkg/bootstrap/registry/templates/registry.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var ( - // RegistryServiceTempl defines the template of registry service for systemd. - RegistryServiceTempl = template.Must(template.New("registryService").Parse( - dedent.Dedent(`[Unit] -Description=v2 Registry server for Container -After=network.target -[Service] -Type=simple -ExecStart=/usr/local/bin/registry serve /etc/kubekey/registry/config.yaml -Restart=on-failure -[Install] -WantedBy=multi-user.target - `))) - - // RegistryConfigTempl defines the template of registry's configuration file. - RegistryConfigTempl = template.Must(template.New("registryConfig").Parse( - dedent.Dedent(`version: 0.1 -log: - fields: - service: registry -storage: - cache: - layerinfo: inmemory - filesystem: - rootdirectory: /mnt/registry -http: - addr: :443 - tls: - certificate: /etc/ssl/registry/ssl/{{ .Certificate }} - key: /etc/ssl/registry/ssl/{{ .Key }} - `))) -) diff --git a/cmd/kk/pkg/certs/module.go b/cmd/kk/pkg/certs/module.go deleted file mode 100644 index 34130b9d..00000000 --- a/cmd/kk/pkg/certs/module.go +++ /dev/null @@ -1,190 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package certs - -import ( - "path/filepath" - - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" -) - -type CheckCertsModule struct { - common.KubeModule -} - -func (c *CheckCertsModule) Init() { - c.Name = "CheckCertsModule" - c.Desc = "Check cluster certs" - - check := &task.RemoteTask{ - Name: "CheckClusterCerts", - Desc: "Check cluster certs", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Action: new(ListClusterCerts), - Parallel: true, - } - - c.Tasks = []task.Interface{ - check, - } -} - -type PrintClusterCertsModule struct { - common.KubeModule -} - -func (p *PrintClusterCertsModule) Init() { - p.Name = "PrintClusterCertsModule" - p.Desc = "Display cluster certs form" - - display := &task.LocalTask{ - Name: "DisplayCertsForm", - Desc: "Display cluster certs form", - Action: new(DisplayForm), - } - - p.Tasks = []task.Interface{ - display, - } -} - -type RenewCertsModule struct { - common.KubeModule -} - -func (r *RenewCertsModule) Init() { - r.Name = "RenewCertsModule" - r.Desc = "Renew control-plane certs" - - renew := &task.RemoteTask{ - Name: "RenewCerts", - Desc: "Renew control-plane certs", - Hosts: r.Runtime.GetHostsByRole(common.Master), - Action: new(RenewCerts), - Parallel: false, - Retry: 5, - } - - copyKubeConfig := &task.RemoteTask{ - Name: "CopyKubeConfig", - Desc: "Copy admin.conf to ~/.kube/config", - Hosts: r.Runtime.GetHostsByRole(common.Master), - Action: new(kubernetes.CopyKubeConfigForControlPlane), - Parallel: true, - Retry: 2, - } - - r.Tasks = []task.Interface{ - renew, - copyKubeConfig, - } -} - -type AutoRenewCertsModule struct { - common.KubeModule - Skip bool -} - -func (a *AutoRenewCertsModule) IsSkip() bool { - return a.Skip -} - -func (a *AutoRenewCertsModule) Init() { - a.Name = "AutoRenewCertsModule" - a.Desc = "Install auto renew control-plane certs" - - generateK8sCertsRenewScript := &task.RemoteTask{ - Name: "GenerateK8sCertsRenewScript", - Desc: "Generate k8s certs renew script", - Hosts: a.Runtime.GetHostsByRole(common.Master), - Action: &action.Template{ - Template: templates.K8sCertsRenewScript, - Dst: filepath.Join("/usr/local/bin/kube-scripts/", templates.K8sCertsRenewScript.Name()), - Data: util.Data{ - "IsDocker": a.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker, - "IsKubeadmAlphaCerts": versionutil.MustParseSemantic(a.KubeConf.Cluster.Kubernetes.Version).LessThan(versionutil.MustParseGeneric("v1.20.0")), - }, - }, - Parallel: true, - } - - generateK8sCertsRenewService := &task.RemoteTask{ - Name: "GenerateK8sCertsRenewService", - Desc: "Generate k8s certs renew service", - Hosts: a.Runtime.GetHostsByRole(common.Master), - Action: &action.Template{ - Template: templates.K8sCertsRenewService, - Dst: filepath.Join("/etc/systemd/system/", templates.K8sCertsRenewService.Name()), - }, - Parallel: true, - } - - generateK8sCertsRenewTimer := &task.RemoteTask{ - Name: "GenerateK8sCertsRenewTimer", - Desc: "Generate k8s certs renew timer", - Hosts: a.Runtime.GetHostsByRole(common.Master), - Action: &action.Template{ - Template: templates.K8sCertsRenewTimer, - Dst: filepath.Join("/etc/systemd/system/", templates.K8sCertsRenewTimer.Name()), - }, - Parallel: true, - } - - enable := &task.RemoteTask{ - Name: "EnableK8sCertsRenewService", - Desc: "Enable k8s certs renew service", - Hosts: a.Runtime.GetHostsByRole(common.Master), - Action: new(EnableRenewService), - Parallel: true, - } - - a.Tasks = []task.Interface{ - generateK8sCertsRenewScript, - generateK8sCertsRenewService, - generateK8sCertsRenewTimer, - enable, - } -} - -type UninstallAutoRenewCertsModule struct { - common.KubeModule -} - -func (u *UninstallAutoRenewCertsModule) Init() { - u.Name = "UninstallAutoRenewCertsModule" - u.Desc = "UnInstall auto renew control-plane certs" - - uninstall := &task.RemoteTask{ - Name: "UnInstallAutoRenewCerts", - Desc: "UnInstall auto renew control-plane certs", - Hosts: u.Runtime.GetHostsByRole(common.Master), - Prepare: new(AutoRenewCertsEnabled), - Action: new(UninstallAutoRenewCerts), - Parallel: true, - } - - u.Tasks = []task.Interface{ - uninstall, - } -} diff --git a/cmd/kk/pkg/certs/prepares.go b/cmd/kk/pkg/certs/prepares.go deleted file mode 100644 index ff8f69d6..00000000 --- a/cmd/kk/pkg/certs/prepares.go +++ /dev/null @@ -1,41 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package certs - -import ( - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type AutoRenewCertsEnabled struct { - common.KubePrepare - Not bool -} - -func (a *AutoRenewCertsEnabled) PreCheck(runtime connector.Runtime) (bool, error) { - exist, err := runtime.GetRunner().FileExist(filepath.Join("/etc/systemd/system/", templates.K8sCertsRenewService.Name())) - if err != nil { - return false, err - } - if exist { - return !a.Not, nil - } - return a.Not, nil -} diff --git a/cmd/kk/pkg/certs/tasks.go b/cmd/kk/pkg/certs/tasks.go deleted file mode 100644 index 83166146..00000000 --- a/cmd/kk/pkg/certs/tasks.go +++ /dev/null @@ -1,409 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package certs - -import ( - "encoding/base64" - "fmt" - "os" - "path/filepath" - "strings" - "text/tabwriter" - "time" - - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/runtime/schema" - versionutil "k8s.io/apimachinery/pkg/util/version" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - clientcmdlatest "k8s.io/client-go/tools/clientcmd/api/latest" - certutil "k8s.io/client-go/util/cert" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -type Certificate struct { - Name string - Expires string - Residual string - AuthorityName string - NodeName string -} - -type CaCertificate struct { - AuthorityName string - Expires string - Residual string - NodeName string -} - -var ( - certificateList = []string{ - "apiserver.crt", - "apiserver-kubelet-client.crt", - "front-proxy-client.crt", - } - caCertificateList = []string{ - "ca.crt", - "front-proxy-ca.crt", - } - kubeConfigList = []string{ - "admin.conf", - "controller-manager.conf", - "scheduler.conf", - } -) - -type ListClusterCerts struct { - common.KubeAction -} - -func (l *ListClusterCerts) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - certificates := make([]*Certificate, 0) - caCertificates := make([]*CaCertificate, 0) - - for _, certFileName := range certificateList { - certPath := filepath.Join(common.KubeCertDir, certFileName) - certContext, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("cat %s", certPath), false) - if err != nil { - return errors.Wrap(err, "get cluster certs failed") - } - if cert, err := getCertInfo(certContext, certFileName, host.GetName()); err != nil { - return err - } else { - certificates = append(certificates, cert) - } - } - for _, kubeConfigFileName := range kubeConfigList { - kubeConfigPath := filepath.Join(common.KubeConfigDir, kubeConfigFileName) - newConfig := clientcmdapi.NewConfig() - kubeconfigBytes, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("cat %s", kubeConfigPath), false) - decoded, _, err := clientcmdlatest.Codec.Decode([]byte(kubeconfigBytes), &schema.GroupVersionKind{Version: clientcmdlatest.Version, Kind: "Config"}, newConfig) - if err != nil { - return err - } - newConfig = decoded.(*clientcmdapi.Config) - for _, a := range newConfig.AuthInfos { - certContextBase64 := a.ClientCertificateData - tmp := base64.StdEncoding.EncodeToString(certContextBase64) - certContext, err := base64.StdEncoding.DecodeString(tmp) - if err != nil { - return err - } - if cert, err := getCertInfo(string(certContext), kubeConfigFileName, host.GetName()); err != nil { - return err - } else { - certificates = append(certificates, cert) - } - } - } - - for _, caCertFileName := range caCertificateList { - certPath := filepath.Join(common.KubeCertDir, caCertFileName) - caCertContext, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("cat %s", certPath), false) - if err != nil { - return errors.Wrap(err, "Failed to get cluster certs") - } - if cert, err := getCaCertInfo(caCertContext, caCertFileName, host.GetName()); err != nil { - return err - } else { - caCertificates = append(caCertificates, cert) - } - } - - host.GetCache().Set(common.Certificate, certificates) - host.GetCache().Set(common.CaCertificate, caCertificates) - return nil -} - -func getCertInfo(certContext, certFileName, nodeName string) (*Certificate, error) { - certs, err1 := certutil.ParseCertsPEM([]byte(certContext)) - if err1 != nil { - return nil, errors.Wrap(err1, "Failed to get cluster certs") - } - var authorityName string - switch certFileName { - case "apiserver.crt": - authorityName = "ca" - case "apiserver-kubelet-client.crt": - authorityName = "ca" - case "front-proxy-client.crt": - authorityName = "front-proxy-ca" - default: - authorityName = "" - } - cert := Certificate{ - Name: certFileName, - Expires: certs[0].NotAfter.Format("Jan 02, 2006 15:04 MST"), - Residual: ResidualTime(certs[0].NotAfter), - AuthorityName: authorityName, - NodeName: nodeName, - } - return &cert, nil -} - -func getCaCertInfo(certContext, certFileName, nodeName string) (*CaCertificate, error) { - certs, err := certutil.ParseCertsPEM([]byte(certContext)) - if err != nil { - return nil, errors.Wrap(err, "Failed to get cluster certs") - } - cert1 := CaCertificate{ - AuthorityName: certFileName, - Expires: certs[0].NotAfter.Format("Jan 02, 2006 15:04 MST"), - Residual: ResidualTime(certs[0].NotAfter), - NodeName: nodeName, - } - return &cert1, nil -} - -func ResidualTime(t time.Time) string { - d := time.Until(t) - if seconds := int(d.Seconds()); seconds < -1 { - return fmt.Sprintf("") - } else if seconds < 0 { - return fmt.Sprintf("0s") - } else if seconds < 60 { - return fmt.Sprintf("%ds", seconds) - } else if minutes := int(d.Minutes()); minutes < 60 { - return fmt.Sprintf("%dm", minutes) - } else if hours := int(d.Hours()); hours < 24 { - return fmt.Sprintf("%dh", hours) - } else if hours < 24*365 { - return fmt.Sprintf("%dd", hours/24) - } - return fmt.Sprintf("%dy", int(d.Hours()/24/365)) -} - -type DisplayForm struct { - common.KubeAction -} - -func (d *DisplayForm) Execute(runtime connector.Runtime) error { - certificates := make([]*Certificate, 0) - caCertificates := make([]*CaCertificate, 0) - - for _, host := range runtime.GetHostsByRole(common.Master) { - certs, ok := host.GetCache().Get(common.Certificate) - if !ok { - return errors.New("get certificate failed by pipeline cache") - } - ca, ok := host.GetCache().Get(common.CaCertificate) - if !ok { - return errors.New("get ca certificate failed by pipeline cache") - } - hostCertificates := certs.([]*Certificate) - hostCaCertificates := ca.([]*CaCertificate) - certificates = append(certificates, hostCertificates...) - caCertificates = append(caCertificates, hostCaCertificates...) - } - - w := tabwriter.NewWriter(os.Stdout, 10, 4, 3, ' ', 0) - _, _ = fmt.Fprintln(w, "CERTIFICATE\tEXPIRES\tRESIDUAL TIME\tCERTIFICATE AUTHORITY\tNODE") - for _, cert := range certificates { - s := fmt.Sprintf("%s\t%s\t%s\t%s\t%-8v", - cert.Name, - cert.Expires, - cert.Residual, - cert.AuthorityName, - cert.NodeName, - ) - - _, _ = fmt.Fprintln(w, s) - continue - } - _, _ = fmt.Fprintln(w) - _, _ = fmt.Fprintln(w, "CERTIFICATE AUTHORITY\tEXPIRES\tRESIDUAL TIME\tNODE") - for _, caCert := range caCertificates { - c := fmt.Sprintf("%s\t%s\t%s\t%-8v", - caCert.AuthorityName, - caCert.Expires, - caCert.Residual, - caCert.NodeName, - ) - - _, _ = fmt.Fprintln(w, c) - continue - } - - _ = w.Flush() - return nil -} - -type RenewCerts struct { - common.KubeAction -} - -func (r *RenewCerts) Execute(runtime connector.Runtime) error { - var kubeadmAlphaList = []string{ - "/usr/local/bin/kubeadm alpha certs renew apiserver", - "/usr/local/bin/kubeadm alpha certs renew apiserver-kubelet-client", - "/usr/local/bin/kubeadm alpha certs renew front-proxy-client", - "/usr/local/bin/kubeadm alpha certs renew admin.conf", - "/usr/local/bin/kubeadm alpha certs renew controller-manager.conf", - "/usr/local/bin/kubeadm alpha certs renew scheduler.conf", - } - - var kubeadmList = []string{ - "/usr/local/bin/kubeadm certs renew apiserver", - "/usr/local/bin/kubeadm certs renew apiserver-kubelet-client", - "/usr/local/bin/kubeadm certs renew front-proxy-client", - "/usr/local/bin/kubeadm certs renew admin.conf", - "/usr/local/bin/kubeadm certs renew controller-manager.conf", - "/usr/local/bin/kubeadm certs renew scheduler.conf", - } - - var restartList = []string{ - "docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f", - "docker ps -af name=k8s_kube-scheduler* -q | xargs --no-run-if-empty docker rm -f", - "docker ps -af name=k8s_kube-controller-manager* -q | xargs --no-run-if-empty docker rm -f", - "systemctl restart kubelet", - } - - version, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubeadm version -o short", true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "kubeadm get version failed") - } - cmp, err := versionutil.MustParseSemantic(version).Compare("v1.20.0") - if err != nil { - return errors.Wrap(errors.WithStack(err), "parse kubeadm version failed") - } - if cmp == -1 { - _, err := runtime.GetRunner().SudoCmd(strings.Join(kubeadmAlphaList, " && "), false) - if err != nil { - return errors.Wrap(err, "kubeadm alpha certs renew failed") - } - } else { - _, err := runtime.GetRunner().SudoCmd(strings.Join(kubeadmList, " && "), false) - if err != nil { - return errors.Wrap(err, "kubeadm alpha certs renew failed") - } - } - - _, err = runtime.GetRunner().SudoCmd(strings.Join(restartList, " && "), false) - if err != nil { - return errors.Wrap(err, "kube-apiserver, kube-schedule, kube-controller-manager or kubelet restart failed") - } - return nil -} - -type FetchKubeConfig struct { - common.KubeAction -} - -func (f *FetchKubeConfig) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - tmpConfigFile := filepath.Join(common.TmpDir, "admin.conf") - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("cp /etc/kubernetes/admin.conf %s", tmpConfigFile), false); err != nil { - return errors.Wrap(errors.WithStack(err), "copy kube config to /tmp/ failed") - } - - host := runtime.RemoteHost() - if err := runtime.GetRunner().Fetch(filepath.Join(runtime.GetWorkDir(), host.GetName(), "admin.conf"), tmpConfigFile); err != nil { - return errors.Wrap(errors.WithStack(err), "fetch kube config file failed") - } - return nil -} - -type SyneKubeConfigToWorker struct { - common.KubeAction -} - -func (s *SyneKubeConfigToWorker) Execute(runtime connector.Runtime) error { - createConfigDirCmd := "mkdir -p /root/.kube" - if _, err := runtime.GetRunner().SudoCmd(createConfigDirCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "create .kube dir failed") - } - - firstMaster := runtime.GetHostsByRole(common.Master)[0] - localFile := filepath.Join(runtime.GetWorkDir(), firstMaster.GetName(), "admin.conf") - if err := runtime.GetRunner().SudoScp(localFile, "/root/.kube/config"); err != nil { - return errors.Wrap(errors.WithStack(err), "sudo scp config file to worker /root/.kube/config failed") - } - - // that doesn't work - //if err := runtime.GetRunner().SudoScp(filepath.Join(runtime.GetWorkDir(), firstMaster.GetName(), "admin.conf"), "$HOME/.kube/config"); err != nil { - // return errors.Wrap(errors.WithStack(err), "sudo scp config file to worker $HOME/.kube/config failed") - //} - - if host := runtime.RemoteHost(); host.GetUser() != "root" { - userConfigDirCmd := "mkdir -p $HOME/.kube" - if _, err := runtime.GetRunner().Cmd(userConfigDirCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "user mkdir $HOME/.kube failed") - } - - getKubeConfigCmdUsr := "cp -f /root/.kube/config $HOME/.kube/config" - if _, err := runtime.GetRunner().SudoCmd(getKubeConfigCmdUsr, false); err != nil { - return errors.Wrap(errors.WithStack(err), "user copy /etc/kubernetes/admin.conf to $HOME/.kube/config failed") - } - - userId, err := runtime.GetRunner().Cmd("echo $(id -u)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user id failed") - } - - userGroupId, err := runtime.GetRunner().Cmd("echo $(id -g)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user group id failed") - } - - chownKubeConfig := fmt.Sprintf("chown -R %s:%s $HOME/.kube", userId, userGroupId) - if _, err := runtime.GetRunner().SudoCmd(chownKubeConfig, false); err != nil { - return errors.Wrap(errors.WithStack(err), "chown user kube config failed") - } - } - return nil -} - -type EnableRenewService struct { - common.KubeAction -} - -func (e *EnableRenewService) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - "chmod +x /usr/local/bin/kube-scripts/k8s-certs-renew.sh && systemctl enable --now k8s-certs-renew.timer", - false); err != nil { - return errors.Wrap(errors.WithStack(err), "enable k8s renew certs service failed") - } - return nil -} - -type UninstallAutoRenewCerts struct { - common.KubeAction -} - -func (u *UninstallAutoRenewCerts) Execute(runtime connector.Runtime) error { - _, _ = runtime.GetRunner().SudoCmd("systemctl disable k8s-certs-renew.timer 1>/dev/null 2>/dev/null", true) - _, _ = runtime.GetRunner().SudoCmd("systemctl stop k8s-certs-renew.timer 1>/dev/null 2>/dev/null", true) - - files := []string{ - filepath.Join("/usr/local/bin/kube-scripts/", templates.K8sCertsRenewScript.Name()), - filepath.Join("/etc/systemd/system/", templates.K8sCertsRenewService.Name()), - filepath.Join("/etc/systemd/system/", templates.K8sCertsRenewTimer.Name()), - } - for _, file := range files { - _, _ = runtime.GetRunner().SudoCmd(fmt.Sprintf("rm -rf %s", file), true) - } - - return nil -} diff --git a/cmd/kk/pkg/certs/templates/certs_renew_script.go b/cmd/kk/pkg/certs/templates/certs_renew_script.go deleted file mode 100644 index e7b590a2..00000000 --- a/cmd/kk/pkg/certs/templates/certs_renew_script.go +++ /dev/null @@ -1,55 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -// K8sCertsRenewScript defines the template of k8s-certs-renew timer for systemd. -var K8sCertsRenewScript = template.Must(template.New("k8s-certs-renew.sh").Parse( - dedent.Dedent(`#!/bin/bash -{{- if .IsKubeadmAlphaCerts }} -kubeadmCerts='/usr/local/bin/kubeadm alpha certs' -{{- else}} -kubeadmCerts='/usr/local/bin/kubeadm certs' -{{- end }} -getCertValidDays() { - local earliestExpireDate; earliestExpireDate=$(${kubeadmCerts} check-expiration | grep -o "[A-Za-z]\{3,4\}\s\w\w,\s[0-9]\{4,\}\s\w*:\w*\s\w*\s*" | xargs -I {} date -d {} +%s | sort | head -n 1) - local today; today="$(date +%s)" - echo -n $(( ($earliestExpireDate - $today) / (24 * 60 * 60) )) -} -echo "## Expiration before renewal ##" -${kubeadmCerts} check-expiration -if [ $(getCertValidDays) -lt 30 ]; then - echo "## Renewing certificates managed by kubeadm ##" - ${kubeadmCerts} renew all - echo "## Restarting control plane pods managed by kubeadm ##" -{{- if .IsDocker}} - $(which docker | grep docker) ps -af 'name=k8s_POD_(kube-apiserver|kube-controller-manager|kube-scheduler|etcd)-*' -q | /usr/bin/xargs $(which docker | grep docker) rm -f -{{- else}} - $(which crictl | grep crictl) pods --namespace kube-system --name 'kube-scheduler-*|kube-controller-manager-*|kube-apiserver-*|etcd-*' -q | /usr/bin/xargs $(which crictl | grep crictl) rmp -f -{{- end }} - echo "## Updating /root/.kube/config ##" - cp /etc/kubernetes/admin.conf /root/.kube/config -fi -echo "## Waiting for apiserver to be up again ##" -until printf "" 2>>/dev/null >>/dev/tcp/127.0.0.1/6443; do sleep 1; done -echo "## Expiration after renewal ##" -${kubeadmCerts} check-expiration - `))) diff --git a/cmd/kk/pkg/certs/templates/certs_renew_service.go b/cmd/kk/pkg/certs/templates/certs_renew_service.go deleted file mode 100644 index b29aa39f..00000000 --- a/cmd/kk/pkg/certs/templates/certs_renew_service.go +++ /dev/null @@ -1,44 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var ( - // K8sCertsRenewService defines the template of k8s-certs-renew service for systemd. - K8sCertsRenewService = template.Must(template.New("k8s-certs-renew.service").Parse( - dedent.Dedent(`[Unit] -Description=Renew K8S control plane certificates -[Service] -Type=oneshot -ExecStart=/usr/local/bin/kube-scripts/k8s-certs-renew.sh - `))) - - // K8sCertsRenewTimer defines the template of k8s-certs-renew timer for systemd. - K8sCertsRenewTimer = template.Must(template.New("k8s-certs-renew.timer").Parse( - dedent.Dedent(`[Unit] -Description=Timer to renew K8S control plane certificates -[Timer] -OnCalendar=Mon *-*-* 03:00:00 -Unit=k8s-certs-renew.service -[Install] -WantedBy=multi-user.target - `))) -) diff --git a/cmd/kk/pkg/client/kubernetes/client.go b/cmd/kk/pkg/client/kubernetes/client.go deleted file mode 100644 index f40a9463..00000000 --- a/cmd/kk/pkg/client/kubernetes/client.go +++ /dev/null @@ -1,43 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - "os" -) - -func NewClient(kubeConfig string) (*kubernetes.Clientset, error) { - data, err := os.ReadFile(kubeConfig) - if err != nil { - return nil, err - } - config, err := clientcmd.NewClientConfigFromBytes(data) - if err != nil { - return nil, err - } - restConfig, err := config.ClientConfig() - if err != nil { - return nil, err - } - client, err := kubernetes.NewForConfig(restConfig) - if err != nil { - return nil, err - } - return client, nil -} diff --git a/cmd/kk/pkg/common/artifact_action.go b/cmd/kk/pkg/common/artifact_action.go deleted file mode 100644 index c2f60c0c..00000000 --- a/cmd/kk/pkg/common/artifact_action.go +++ /dev/null @@ -1,37 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type ArtifactAction struct { - action.BaseAction - Manifest *ArtifactManifest -} - -func (a *ArtifactAction) AutoAssert(runtime connector.Runtime) { - artifactRuntime := runtime.(*ArtifactRuntime) - m := &ArtifactManifest{ - Spec: artifactRuntime.Spec, - Arg: artifactRuntime.Arg, - } - - a.Manifest = m -} diff --git a/cmd/kk/pkg/common/artifact_module.go b/cmd/kk/pkg/common/artifact_module.go deleted file mode 100644 index aa850f0c..00000000 --- a/cmd/kk/pkg/common/artifact_module.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - kubekeyv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" -) - -type ArtifactManifest struct { - Spec *kubekeyv1alpha2.ManifestSpec - Arg ArtifactArgument -} - -type ArtifactModule struct { - module.BaseTaskModule - Manifest *ArtifactManifest -} - -func (a *ArtifactModule) IsSkip() bool { - return a.Skip -} - -func (a *ArtifactModule) AutoAssert() { - artifactRuntime := a.Runtime.(*ArtifactRuntime) - m := &ArtifactManifest{ - Spec: artifactRuntime.Spec, - Arg: artifactRuntime.Arg, - } - - a.Manifest = m -} diff --git a/cmd/kk/pkg/common/artifact_prepare.go b/cmd/kk/pkg/common/artifact_prepare.go deleted file mode 100644 index 92423e29..00000000 --- a/cmd/kk/pkg/common/artifact_prepare.go +++ /dev/null @@ -1,37 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" -) - -type ArtifactPrepare struct { - prepare.BasePrepare - Manifest *ArtifactManifest -} - -func (a *ArtifactPrepare) AutoAssert(runtime connector.Runtime) { - artifactRuntime := runtime.(*ArtifactRuntime) - m := &ArtifactManifest{ - Spec: artifactRuntime.Spec, - Arg: artifactRuntime.Arg, - } - - a.Manifest = m -} diff --git a/cmd/kk/pkg/common/artifact_runtime.go b/cmd/kk/pkg/common/artifact_runtime.go deleted file mode 100644 index fdd453bf..00000000 --- a/cmd/kk/pkg/common/artifact_runtime.go +++ /dev/null @@ -1,84 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - "encoding/json" - "os" - "path/filepath" - - "github.com/pkg/errors" - k8syaml "k8s.io/apimachinery/pkg/util/yaml" - - kubekeyv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type ArtifactArgument struct { - ManifestFile string - Output string - CriSocket string - Debug bool - IgnoreErr bool - DownloadCommand func(path, url string) string -} - -type ArtifactRuntime struct { - LocalRuntime - Spec *kubekeyv1alpha2.ManifestSpec - Arg ArtifactArgument -} - -func NewArtifactRuntime(arg ArtifactArgument) (*ArtifactRuntime, error) { - localRuntime, err := NewLocalRuntime(arg.Debug, arg.IgnoreErr) - if err != nil { - return nil, err - } - - fp, err := filepath.Abs(arg.ManifestFile) - if err != nil { - return nil, errors.Wrap(err, "Failed to look up current directory") - } - - fileByte, err := os.ReadFile(fp) - if err != nil { - return nil, errors.Wrapf(err, "Failed to read file %s", fp) - } - - contentToJson, err := k8syaml.ToJSON(fileByte) - if err != nil { - return nil, errors.Wrap(err, "Unable to convert configuration to json") - } - - manifest := &kubekeyv1alpha2.Manifest{} - if err := json.Unmarshal(contentToJson, manifest); err != nil { - return nil, errors.Wrapf(err, "Failed to json unmarshal") - } - - r := &ArtifactRuntime{ - Spec: &manifest.Spec, - Arg: arg, - } - r.LocalRuntime = localRuntime - return r, nil -} - -// Copy is used to create a copy for Runtime. -func (a *ArtifactRuntime) Copy() connector.Runtime { - runtime := *a - return &runtime -} diff --git a/cmd/kk/pkg/common/common.go b/cmd/kk/pkg/common/common.go deleted file mode 100644 index 4b24d7a3..00000000 --- a/cmd/kk/pkg/common/common.go +++ /dev/null @@ -1,98 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -const ( - K3s = "k3s" - K8e = "k8e" - Kubernetes = "kubernetes" - - LocalHost = "localhost" - - AllInOne = "allInOne" - File = "file" - Operator = "operator" - - Master = "master" - Worker = "worker" - ETCD = "etcd" - K8s = "k8s" - Registry = "registry" - KubeKey = "kubekey" - Harbor = "harbor" - DockerCompose = "compose" - - KubeBinaries = "KubeBinaries" - - TmpDir = "/tmp/kubekey" - BinDir = "/usr/local/bin" - KubeConfigDir = "/etc/kubernetes" - KubeAddonsDir = "/etc/kubernetes/addons" - KubeCertDir = "/etc/kubernetes/pki" - KubeManifestDir = "/etc/kubernetes/manifests" - KubeScriptDir = "/usr/local/bin/kube-scripts" - KubeletFlexvolumesPluginsDir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec" - - ETCDCertDir = "/etc/ssl/etcd/ssl" - RegistryCertDir = "/etc/ssl/registry/ssl" - - HaproxyDir = "/etc/kubekey/haproxy" - - IPv4Regexp = "[\\d]+\\.[\\d]+\\.[\\d]+\\.[\\d]+" - IPv6Regexp = "[a-f0-9]{1,4}(:[a-f0-9]{1,4}){7}|[a-f0-9]{1,4}(:[a-f0-9]{1,4}){0,7}::[a-f0-9]{0,4}(:[a-f0-9]{1,4}){0,7}" - - Calico = "calico" - Flannel = "flannel" - Cilium = "cilium" - Kubeovn = "kubeovn" - Hybridnet = "hybridnet" - - Docker = "docker" - Crictl = "crictl" - Containerd = "containerd" - Crio = "crio" - Isula = "isula" - Runc = "runc" - - // global cache key - // PreCheckModule - NodePreCheck = "nodePreCheck" - K8sVersion = "k8sVersion" // current k8s version - MaxK8sVersion = "maxK8sVersion" // max k8s version of nodes - KubeSphereVersion = "kubeSphereVersion" // current KubeSphere version - ClusterNodeStatus = "clusterNodeStatus" - ClusterNodeCRIRuntimes = "ClusterNodeCRIRuntimes" - DesiredK8sVersion = "desiredK8sVersion" - PlanK8sVersion = "planK8sVersion" - NodeK8sVersion = "NodeK8sVersion" - - // ETCDModule - ETCDCluster = "etcdCluster" - ETCDName = "etcdName" - ETCDExist = "etcdExist" - - // KubernetesModule - ClusterStatus = "clusterStatus" - ClusterExist = "clusterExist" - - // CertsModule - Certificate = "certificate" - CaCertificate = "caCertificate" - - // Artifact pipeline - Artifact = "artifact" -) diff --git a/cmd/kk/pkg/common/kube_action.go b/cmd/kk/pkg/common/kube_action.go deleted file mode 100644 index 346d4dc9..00000000 --- a/cmd/kk/pkg/common/kube_action.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type KubeAction struct { - action.BaseAction - KubeConf *KubeConf -} - -func (k *KubeAction) AutoAssert(runtime connector.Runtime) { - kubeRuntime := runtime.(*KubeRuntime) - conf := &KubeConf{ - Cluster: kubeRuntime.Cluster, - ClusterName: kubeRuntime.ClusterName, - Kubeconfig: kubeRuntime.Kubeconfig, - Arg: kubeRuntime.Arg, - } - - k.KubeConf = conf -} diff --git a/cmd/kk/pkg/common/kube_module.go b/cmd/kk/pkg/common/kube_module.go deleted file mode 100644 index 38f4191a..00000000 --- a/cmd/kk/pkg/common/kube_module.go +++ /dev/null @@ -1,68 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" -) - -type KubeConf struct { - ClusterHosts []string - ClusterName string - Cluster *kubekeyapiv1alpha2.ClusterSpec - Kubeconfig string - Arg Argument -} - -type KubeModule struct { - module.BaseTaskModule - KubeConf *KubeConf -} - -func (k *KubeModule) IsSkip() bool { - return k.Skip -} - -func (k *KubeModule) AutoAssert() { - kubeRuntime := k.Runtime.(*KubeRuntime) - conf := &KubeConf{ - ClusterName: kubeRuntime.ClusterName, - Cluster: kubeRuntime.Cluster, - Kubeconfig: kubeRuntime.Kubeconfig, - Arg: kubeRuntime.Arg, - } - - k.KubeConf = conf -} - -type KubeCustomModule struct { - module.CustomModule - KubeConf *KubeConf -} - -func (k *KubeCustomModule) AutoAssert() { - kubeRuntime := k.Runtime.(*KubeRuntime) - conf := &KubeConf{ - ClusterName: kubeRuntime.ClusterName, - Cluster: kubeRuntime.Cluster, - Kubeconfig: kubeRuntime.Kubeconfig, - Arg: kubeRuntime.Arg, - } - - k.KubeConf = conf -} diff --git a/cmd/kk/pkg/common/kube_prepare.go b/cmd/kk/pkg/common/kube_prepare.go deleted file mode 100644 index 0ee6b983..00000000 --- a/cmd/kk/pkg/common/kube_prepare.go +++ /dev/null @@ -1,138 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" -) - -type KubePrepare struct { - prepare.BasePrepare - KubeConf *KubeConf -} - -func (k *KubePrepare) AutoAssert(runtime connector.Runtime) { - kubeRuntime := runtime.(*KubeRuntime) - conf := &KubeConf{ - Cluster: kubeRuntime.Cluster, - Kubeconfig: kubeRuntime.Kubeconfig, - Arg: kubeRuntime.Arg, - } - - k.KubeConf = conf -} - -type OnlyFirstMaster struct { - KubePrepare - Not bool -} - -func (o *OnlyFirstMaster) PreCheck(runtime connector.Runtime) (bool, error) { - if runtime.RemoteHost().IsRole(Master) && - runtime.RemoteHost().GetName() == runtime.GetHostsByRole(Master)[0].GetName() { - return !o.Not, nil - } - return o.Not, nil -} - -type IsMaster struct { - KubePrepare -} - -func (i *IsMaster) PreCheck(runtime connector.Runtime) (bool, error) { - if runtime.RemoteHost().IsRole(Master) { - return true, nil - } - return false, nil -} - -type IsWorker struct { - KubePrepare - Not bool -} - -func (i *IsWorker) PreCheck(runtime connector.Runtime) (bool, error) { - if runtime.RemoteHost().IsRole(Worker) { - return !i.Not, nil - } - return i.Not, nil -} - -type OnlyWorker struct { - KubePrepare -} - -func (o *OnlyWorker) PreCheck(runtime connector.Runtime) (bool, error) { - if runtime.RemoteHost().IsRole(Worker) && !runtime.RemoteHost().IsRole(Master) { - return true, nil - } - return false, nil -} - -type OnlyETCD struct { - KubePrepare - Not bool -} - -func (o *OnlyETCD) PreCheck(runtime connector.Runtime) (bool, error) { - if runtime.RemoteHost().IsRole(ETCD) { - return !o.Not, nil - } - return o.Not, nil -} - -type OnlyK3s struct { - KubePrepare -} - -func (o *OnlyK3s) PreCheck(_ connector.Runtime) (bool, error) { - if o.KubeConf.Cluster.Kubernetes.Type == "k3s" { - return true, nil - } - return false, nil -} - -type OnlyKubernetes struct { - KubePrepare -} - -func (o *OnlyKubernetes) PreCheck(_ connector.Runtime) (bool, error) { - if o.KubeConf.Cluster.Kubernetes.Type != "k3s" { - return true, nil - } - return false, nil -} - -type EnableKubeProxy struct { - KubePrepare -} - -func (e *EnableKubeProxy) PreCheck(_ connector.Runtime) (bool, error) { - if !e.KubeConf.Cluster.Kubernetes.DisableKubeProxy { - return true, nil - } - return false, nil -} - -type EnableAudit struct { - KubePrepare -} - -func (e *EnableAudit) PreCheck(_ connector.Runtime) (bool, error) { - return e.KubeConf.Cluster.Kubernetes.EnableAudit(), nil -} diff --git a/cmd/kk/pkg/common/kube_rollback.go b/cmd/kk/pkg/common/kube_rollback.go deleted file mode 100644 index 00dee4c5..00000000 --- a/cmd/kk/pkg/common/kube_rollback.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/rollback" -) - -type KubeRollback struct { - rollback.BaseRollback - KubeConf *KubeConf -} - -func (k *KubeRollback) AutoAssert(runtime connector.Runtime) { - kubeRuntime := runtime.(*KubeRuntime) - conf := &KubeConf{ - Cluster: kubeRuntime.Cluster, - ClusterName: kubeRuntime.ClusterName, - Kubeconfig: kubeRuntime.Kubeconfig, - Arg: kubeRuntime.Arg, - } - - k.KubeConf = conf -} diff --git a/cmd/kk/pkg/common/kube_runtime.go b/cmd/kk/pkg/common/kube_runtime.go deleted file mode 100644 index 970e924e..00000000 --- a/cmd/kk/pkg/common/kube_runtime.go +++ /dev/null @@ -1,100 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type KubeRuntime struct { - connector.BaseRuntime - ClusterName string - Cluster *kubekeyapiv1alpha2.ClusterSpec - Kubeconfig string - Arg Argument -} - -type Argument struct { - NodeName string - FilePath string - KubernetesVersion string - KsEnable bool - KsVersion string - Debug bool - IgnoreErr bool - SkipPullImages bool - SkipPushImages bool - SecurityEnhancement bool - DeployLocalStorage *bool - DownloadCommand func(path, url string) string - SkipConfirmCheck bool - ContainerManager string - FromCluster bool - KubeConfig string - Artifact string - InstallPackages bool - ImagesDir string - Namespace string - DeleteCRI bool - Role string - Type string -} - -func NewKubeRuntime(flag string, arg Argument) (*KubeRuntime, error) { - loader := NewLoader(flag, arg) - cluster, err := loader.Load() - if err != nil { - return nil, err - } - - base := connector.NewBaseRuntime(cluster.Name, connector.NewDialer(), arg.Debug, arg.IgnoreErr) - - clusterSpec := &cluster.Spec - defaultCluster, roleGroups := clusterSpec.SetDefaultClusterSpec() - - hostSet := make(map[string]struct{}) - for _, role := range roleGroups { - for _, host := range role { - if host.IsRole(Master) || host.IsRole(Worker) { - host.SetRole(K8s) - } - if _, ok := hostSet[host.GetName()]; !ok { - hostSet[host.GetName()] = struct{}{} - base.AppendHost(host) - base.AppendRoleMap(host) - } - } - } - - arg.KsEnable = defaultCluster.KubeSphere.Enabled - arg.KsVersion = defaultCluster.KubeSphere.Version - r := &KubeRuntime{ - Cluster: defaultCluster, - ClusterName: cluster.Name, - Arg: arg, - } - r.BaseRuntime = base - - return r, nil -} - -// Copy is used to create a copy for Runtime. -func (k *KubeRuntime) Copy() connector.Runtime { - runtime := *k - return &runtime -} diff --git a/cmd/kk/pkg/common/loader.go b/cmd/kk/pkg/common/loader.go deleted file mode 100644 index 06098c03..00000000 --- a/cmd/kk/pkg/common/loader.go +++ /dev/null @@ -1,308 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - "bufio" - "encoding/json" - "fmt" - "os" - "os/exec" - "os/user" - "path/filepath" - "regexp" - "runtime" - "strings" - "time" - - "github.com/pkg/errors" - "gopkg.in/yaml.v3" - k8syaml "k8s.io/apimachinery/pkg/util/yaml" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -var ( - kubeReleaseRegex = regexp.MustCompile(`^v?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)([-0-9a-zA-Z_\.+]*)?$`) -) - -type Loader interface { - Load() (*kubekeyapiv1alpha2.Cluster, error) -} - -type Options map[string]interface{} - -func NewLoader(flag string, arg Argument) Loader { - switch flag { - case File: - return NewFileLoader(arg) - case Operator: - return &ConfigMapLoader{} - case AllInOne: - return NewDefaultLoader(arg) - default: - return NewDefaultLoader(arg) - } -} - -type DefaultLoader struct { - arg Argument - KubernetesVersion string - KubeSphereVersion string - KubeSphereEnable bool -} - -func NewDefaultLoader(arg Argument) *DefaultLoader { - return &DefaultLoader{ - arg: arg, - KubernetesVersion: arg.KubernetesVersion, - KubeSphereVersion: arg.KsVersion, - KubeSphereEnable: arg.KsEnable, - } -} - -func (d *DefaultLoader) Load() (*kubekeyapiv1alpha2.Cluster, error) { - u, _ := user.Current() - if u.Username != "root" { - return nil, errors.New(fmt.Sprintf("Current user is %s. Please use root!", u.Username)) - } - - allInOne := kubekeyapiv1alpha2.Cluster{} - if output, err := exec.Command("/bin/sh", "-c", "if [ ! -f \"$HOME/.ssh/id_rsa\" ]; then ssh-keygen -t rsa-sha2-512 -P \"\" -f $HOME/.ssh/id_rsa && ls $HOME/.ssh;fi;").CombinedOutput(); err != nil { - return nil, errors.New(fmt.Sprintf("Failed to generate public key: %v\n%s", err, string(output))) - } - if output, err := exec.Command("/bin/sh", "-c", "echo \"\n$(cat $HOME/.ssh/id_rsa.pub)\" >> $HOME/.ssh/authorized_keys && awk ' !x[$0]++{print > \"'$HOME'/.ssh/authorized_keys\"}' $HOME/.ssh/authorized_keys").CombinedOutput(); err != nil { - return nil, errors.New(fmt.Sprintf("Failed to copy public key to authorized_keys: %v\n%s", err, string(output))) - } - - hostname, err := os.Hostname() - if err != nil { - return nil, errors.New(fmt.Sprintf("Failed to get hostname: %v\n", err)) - } - - allInOne.Spec.Hosts = append(allInOne.Spec.Hosts, kubekeyapiv1alpha2.HostCfg{ - Name: hostname, - Address: util.LocalIP(), - InternalAddress: util.LocalIP(), - Port: kubekeyapiv1alpha2.DefaultSSHPort, - User: u.Name, - Password: "", - PrivateKeyPath: fmt.Sprintf("%s/.ssh/id_rsa", u.HomeDir), - Arch: runtime.GOARCH, - }) - - allInOne.Spec.RoleGroups = map[string][]string{ - Master: {hostname}, - ETCD: {hostname}, - Worker: {hostname}, - Registry: {hostname}, - } - allInOne.Spec.ControlPlaneEndpoint.Address = "127.0.0.1" - if ver := normalizedBuildVersion(d.KubernetesVersion); ver != "" { - s := strings.Split(ver, "-") - if len(s) > 1 { - allInOne.Spec.Kubernetes = kubekeyapiv1alpha2.Kubernetes{ - Version: s[0], - Type: s[1], - } - } else { - allInOne.Spec.Kubernetes = kubekeyapiv1alpha2.Kubernetes{ - Version: ver, - } - } - } else { - allInOne.Spec.Kubernetes = kubekeyapiv1alpha2.Kubernetes{ - Version: kubekeyapiv1alpha2.DefaultKubeVersion, - } - } - - if d.KubeSphereEnable { - ver := normalizedBuildVersion(d.KubeSphereVersion) - if ver == "" { - return nil, errors.New(fmt.Sprintf("Unsupported Kubesphere Version: %v\n", d.KubeSphereVersion)) - } - if err := defaultKSConfig(&allInOne.Spec.KubeSphere, ver); err != nil { - return nil, err - } - } - - if d.arg.ContainerManager != "" && d.arg.ContainerManager != Docker { - allInOne.Spec.Kubernetes.ContainerManager = d.arg.ContainerManager - } - - enableAutoRenewCerts := true - allInOne.Spec.Kubernetes.AutoRenewCerts = &enableAutoRenewCerts - - // must be a lower case - allInOne.Name = "kubekey" + time.Now().Format("2006-01-02") - return &allInOne, nil -} - -type FileLoader struct { - arg Argument - FilePath string - KubernetesVersion string - KubeSphereVersion string - KubeSphereEnable bool -} - -func NewFileLoader(arg Argument) *FileLoader { - return &FileLoader{ - arg: arg, - FilePath: arg.FilePath, - KubernetesVersion: arg.KubernetesVersion, - KubeSphereVersion: arg.KsVersion, - KubeSphereEnable: arg.KsEnable, - } -} - -func (f FileLoader) Load() (*kubekeyapiv1alpha2.Cluster, error) { - var objName string - - clusterCfg := kubekeyapiv1alpha2.Cluster{} - fp, err := filepath.Abs(f.FilePath) - if err != nil { - return nil, errors.Wrap(err, "Failed to look up current directory") - } - // fixme: It will lead to nil pointer err - //if len(f.KubernetesVersion) != 0 { - // _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("sed -i \"/version/s/\\:.*/\\: %s/g\" %s", f.KubernetesVersion, fp)).Run() - //} - file, err := os.Open(fp) - if err != nil { - return nil, errors.Wrap(err, "Unable to open the given cluster configuration file") - } - defer file.Close() - b1 := bufio.NewReader(file) - for { - result := make(map[string]interface{}) - content, err := k8syaml.NewYAMLReader(b1).Read() - if len(content) == 0 { - break - } - if err != nil { - return nil, errors.Wrap(err, "Unable to read the given cluster configuration file") - } - err = yaml.Unmarshal(content, &result) - if err != nil { - return nil, errors.Wrap(err, "Unable to unmarshal the given cluster configuration file") - } - - if result["kind"] == "Cluster" { - contentToJson, err := k8syaml.ToJSON(content) - if err != nil { - return nil, errors.Wrap(err, "Unable to convert configuration to json") - } - if err := json.Unmarshal(contentToJson, &clusterCfg); err != nil { - return nil, errors.Wrap(err, "Failed to unmarshal configuration") - } - metadata := result["metadata"].(map[string]interface{}) - objName = metadata["name"].(string) - } - - if result["kind"] == "ConfigMap" || result["kind"] == "ClusterConfiguration" { - metadata := result["metadata"].(map[string]interface{}) - labels := metadata["labels"].(map[string]interface{}) - clusterCfg.Spec.KubeSphere.Enabled = true - - v, ok := labels["version"] - if !ok { - return nil, errors.New("Unknown version") - } - - version := v.(string) - _, stable := kubesphere.StabledVersionSupport(version) - _, latest := kubesphere.LatestRelease(version) - _, dev := kubesphere.DevRelease(version) - if stable || latest || dev { - clusterCfg.Spec.KubeSphere.Configurations = "---\n" + string(content) - clusterCfg.Spec.KubeSphere.Version = version - } else { - return nil, errors.New(fmt.Sprintf("Unsupported KubeSphere version: %s", version)) - } - } - } - - if f.KubeSphereEnable { - ver := normalizedBuildVersion(f.KubeSphereVersion) - if ver == "" { - return nil, errors.New(fmt.Sprintf("Unsupported Kubesphere Version: %v\n", f.KubeSphereVersion)) - } - if err := defaultKSConfig(&clusterCfg.Spec.KubeSphere, ver); err != nil { - return nil, err - } - } - - if ver := normalizedBuildVersion(f.KubernetesVersion); ver != "" { - s := strings.Split(ver, "-") - if len(s) > 1 { - clusterCfg.Spec.Kubernetes.Version = s[0] - clusterCfg.Spec.Kubernetes.Type = s[1] - } else { - clusterCfg.Spec.Kubernetes.Version = ver - } - } - - if f.arg.ContainerManager != "" && f.arg.ContainerManager != Docker { - clusterCfg.Spec.Kubernetes.ContainerManager = f.arg.ContainerManager - } - - clusterCfg.Spec.Kubernetes.Version = normalizedBuildVersion(clusterCfg.Spec.Kubernetes.Version) - clusterCfg.Spec.KubeSphere.Version = normalizedBuildVersion(clusterCfg.Spec.KubeSphere.Version) - clusterCfg.Name = objName - return &clusterCfg, nil -} - -type ConfigMapLoader struct { -} - -func (c ConfigMapLoader) Load() (*kubekeyapiv1alpha2.Cluster, error) { - return nil, nil -} - -func defaultKSConfig(ks *kubekeyapiv1alpha2.KubeSphere, version string) error { - ks.Enabled = true - version = strings.TrimSpace(version) - ksInstaller, ok := kubesphere.StabledVersionSupport(version) - if ok { - ks.Version = ksInstaller.Version - ks.Configurations = ksInstaller.CCToString() - } else if latest, ok := kubesphere.LatestRelease(version); ok { - ks.Version = version - ks.Configurations = latest.CCToString() - } else if dev, ok := kubesphere.DevRelease(version); ok { - ks.Version = version - ks.Configurations = dev.CCToString() - } else { - return errors.New(fmt.Sprintf("Unsupported KubeSphere version: %s", version)) - } - return nil -} - -// normalizedBuildVersion used to returns normalized build version (with "v" prefix if needed) -// If input doesn't match known version pattern, returns empty string. -func normalizedBuildVersion(version string) string { - if kubeReleaseRegex.MatchString(version) { - if strings.HasPrefix(version, "v") { - return version - } - return "v" + version - } - return "" -} diff --git a/cmd/kk/pkg/common/local_runtime.go b/cmd/kk/pkg/common/local_runtime.go deleted file mode 100644 index 7902e7f4..00000000 --- a/cmd/kk/pkg/common/local_runtime.go +++ /dev/null @@ -1,75 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -import ( - "fmt" - "os" - "os/exec" - "os/user" - "runtime" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type LocalRuntime struct { - connector.BaseRuntime -} - -func NewLocalRuntime(debug, ingoreErr bool) (LocalRuntime, error) { - var localRuntime LocalRuntime - u, err := user.Current() - if err != nil { - return localRuntime, err - } - if u.Username != "root" { - return localRuntime, fmt.Errorf("current user is %s. Please use root", u.Username) - } - - if output, err := exec.Command("/bin/sh", "-c", "if [ ! -f \"$HOME/.ssh/id_rsa\" ]; then ssh-keygen -t rsa-sha2-512 -P \"\" -f $HOME/.ssh/id_rsa && ls $HOME/.ssh;fi;").CombinedOutput(); err != nil { - return localRuntime, errors.New(fmt.Sprintf("Failed to generate public key: %v\n%s", err, string(output))) - } - if output, err := exec.Command("/bin/sh", "-c", "echo \"\n$(cat $HOME/.ssh/id_rsa.pub)\" >> $HOME/.ssh/authorized_keys && awk ' !x[$0]++{print > \"'$HOME'/.ssh/authorized_keys\"}' $HOME/.ssh/authorized_keys").CombinedOutput(); err != nil { - return localRuntime, errors.New(fmt.Sprintf("Failed to copy public key to authorized_keys: %v\n%s", err, string(output))) - } - - name, err := os.Hostname() - if err != nil { - return localRuntime, err - } - base := connector.NewBaseRuntime(name, connector.NewDialer(), debug, ingoreErr) - - host := connector.NewHost() - host.Name = name - host.Address = util.LocalIP() - host.InternalAddress = util.LocalIP() - host.Port = 22 - host.User = u.Name - host.Password = "" - host.PrivateKeyPath = fmt.Sprintf("%s/.ssh/id_rsa", u.HomeDir) - host.Arch = runtime.GOARCH - host.SetRole(KubeKey) - - base.AppendHost(host) - base.AppendRoleMap(host) - - local := LocalRuntime{base} - return local, nil -} diff --git a/cmd/kk/pkg/config/from_cluster.go b/cmd/kk/pkg/config/from_cluster.go deleted file mode 100644 index faa63b38..00000000 --- a/cmd/kk/pkg/config/from_cluster.go +++ /dev/null @@ -1,281 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "bytes" - "context" - "encoding/base64" - "fmt" - "net" - "os" - "os/exec" - "path/filepath" - "strings" - "text/template" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - "github.com/spf13/viper" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -var ( - // ClusterCfgTempl defines the template of cluster configuration file for the existing cluster. - ClusterCfgTempl = template.Must(template.New("ClusterCfg").Parse( - dedent.Dedent(`apiVersion: kubekey.kubesphere.io/v1alpha2 -kind: Cluster -metadata: - name: {{ .Options.Name }} -spec: - hosts: - ##You should complete the ssh information of the hosts - {{- range .Options.Hosts }} - - {{ . }} - {{- end }} - roleGroups: - etcd: - - SHOULD_BE_REPLACED - master: - {{- range .Options.MasterGroup }} - - {{ . }} - {{- end }} - worker: - {{- range .Options.WorkerGroup }} - - {{ . }} - {{- end }} - controlPlaneEndpoint: - ##Internal loadbalancer for apiservers - {{- if .Options.InternalLoadbalancer }} - internalLoadbalancer: {{ .Options.InternalLoadbalancer }} - {{- else }} - #internalLoadbalancer: haproxy - {{- end }} - - ##If the external loadbalancer was used, 'address' should be set to loadbalancer's ip. - domain: {{ .Options.ControlPlaneEndpointDomain }} - address: {{ .Options.ControlPlaneEndpointAddress }} - port: {{ .Options.ControlPlaneEndpointPort }} - kubernetes: - version: {{ .Options.KubeVersion }} - clusterName: {{ .Options.ClusterName }} - proxyMode: {{ .Options.ProxyMode }} - masqueradeAll: {{ .Options.MasqueradeAll }} - maxPods: {{ .Options.MaxPods }} - nodeCidrMaskSize: {{ .Options.NodeCidrMaskSize }} - network: - plugin: {{ .Options.NetworkPlugin }} - kubePodsCIDR: {{ .Options.PodNetworkCidr }} - kubeServiceCIDR: {{ .Options.ServiceNetworkCidr }} - registry: - privateRegistry: "" - - `))) -) - -// GenerateClusterCfgStr is used to generate cluster configuration content. -func GenerateClusterCfgStr(opt *OptionsCluster) (string, error) { - return util.Render(ClusterCfgTempl, util.Data{ - "Options": opt, - }) -} - -// OptionsCluster defineds the parameters of cluster configuration for the existing cluster. -type OptionsCluster struct { - Name string - Hosts []string - MasterGroup []string - WorkerGroup []string - KubeVersion string - ImageRepo string - ClusterName string - MasqueradeAll string - ProxyMode string - MaxPods string - NodeCidrMaskSize string - PodNetworkCidr string - ServiceNetworkCidr string - NetworkPlugin string - ControlPlaneEndpointDomain string - ControlPlaneEndpointAddress string - ControlPlaneEndpointPort string - InternalLoadbalancer string -} - -// GetInfoFromCluster is used to fetch information from the existing cluster. -func GetInfoFromCluster(config, name string) (*OptionsCluster, error) { - clientset, err := utils.NewClient(config) - if err != nil { - return nil, err - } - - nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, err - } - - opt := OptionsCluster{} - if name != "" { - output := strings.Split(name, ".") - opt.Name = output[0] - } else { - opt.Name = "config-sample" - } - - for _, node := range nodes.Items { - nodeCfg := kubekeyapiv1alpha2.HostCfg{} - nodeInfo, err := clientset.CoreV1().Nodes().Get(context.TODO(), node.Name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - for _, address := range nodeInfo.Status.Addresses { - if address.Type == "Hostname" { - nodeCfg.Name = address.Address - } - if address.Type == "InternalIP" { - nodeCfg.Address = address.Address - nodeCfg.InternalAddress = address.Address - } - } - if _, ok := nodeInfo.Labels["node-role.kubernetes.io/master"]; ok { - opt.MasterGroup = append(opt.MasterGroup, nodeCfg.Name) - if _, ok := nodeInfo.Labels["node-role.kubernetes.io/worker"]; ok { - opt.WorkerGroup = append(opt.WorkerGroup, nodeCfg.Name) - } - } else { - opt.WorkerGroup = append(opt.WorkerGroup, nodeCfg.Name) - } - nodeCfgStr := fmt.Sprintf("{name: %s, address: %s, internalAddress: %s}", nodeCfg.Name, nodeCfg.Address, nodeCfg.InternalAddress) - opt.Hosts = append(opt.Hosts, nodeCfgStr) - - opt.MaxPods = nodeInfo.Status.Capacity.Pods().String() - } - - kubeadmConfig, err := clientset.CoreV1().ConfigMaps("kube-system").Get(context.TODO(), "kubeadm-config", metav1.GetOptions{}) - if err != nil { - return nil, err - } - - viper.SetConfigType("yaml") - if err := viper.ReadConfig(bytes.NewBuffer([]byte(kubeadmConfig.Data["ClusterConfiguration"]))); err != nil { - return nil, err - } - opt.KubeVersion = viper.GetString("kubernetesVersion") - opt.ImageRepo = viper.GetString("imageRepository") - opt.ClusterName = viper.GetString("clusterName") - opt.PodNetworkCidr = viper.GetString("networking.podSubnet") - opt.ServiceNetworkCidr = viper.GetString("networking.serviceSubnet") - if viper.IsSet("controllerManager.extraArgs.node-cidr-mask-size") { - opt.NodeCidrMaskSize = viper.GetString("controllerManager.extraArgs.node-cidr-mask-size") - } else { - opt.NodeCidrMaskSize = "24" - } - if viper.IsSet("controlPlaneEndpoint") { - controlPlaneEndpointStr := viper.GetString("controlPlaneEndpoint") - strList := strings.Split(controlPlaneEndpointStr, ":") - opt.ControlPlaneEndpointPort = strList[len(strList)-1] - strList = strList[:len(strList)-1] - address := strings.Join(strList, ":") - ip := net.ParseIP(address) - if ip != nil { - opt.ControlPlaneEndpointAddress = address - opt.ControlPlaneEndpointDomain = "lb.kubesphere.local" - } else { - opt.ControlPlaneEndpointAddress = "\"\"" - opt.ControlPlaneEndpointDomain = address - } - } - - pods, err := clientset.CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, err - } - for _, pod := range pods.Items { - if strings.Contains(pod.Name, "calico") { - opt.NetworkPlugin = "calico" - } - if strings.Contains(pod.Name, "flannel") { - opt.NetworkPlugin = "flannel" - } - if strings.Contains(pod.Name, "haproxy-node") { - opt.InternalLoadbalancer = "haproxy" - } - } - - kubeProxyConfig, err := clientset.CoreV1().ConfigMaps("kube-system").Get(context.TODO(), "kube-proxy", metav1.GetOptions{}) - if err != nil { - return nil, err - } - - viper.SetConfigType("yaml") - if err := viper.ReadConfig(bytes.NewBuffer([]byte(kubeProxyConfig.Data["config.conf"]))); err != nil { - return nil, err - } - - opt.MasqueradeAll = viper.GetString("iptables.masqueradeAll") - if viper.GetString("mode") == "ipvs" { - opt.ProxyMode = viper.GetString("mode") - } else { - opt.ProxyMode = "iptables" - } - - return &opt, nil -} - -// GenerateConfigFromCluster is used to generate cluster configuration file from the existing cluster's information. -func GenerateConfigFromCluster(cfgPath, kubeconfig, name string) error { - opt, err := GetInfoFromCluster(kubeconfig, name) - if err != nil { - return err - } - - ClusterCfgStr, err := GenerateClusterCfgStr(opt) - if err != nil { - return errors.Wrap(err, "Failed to generate cluster config") - } - ClusterCfgStrBase64 := base64.StdEncoding.EncodeToString([]byte(ClusterCfgStr)) - var configPath string - - if cfgPath != "" { - CheckConfigFileStatus(cfgPath) - cmd := fmt.Sprintf("echo %s | base64 -d > %s", ClusterCfgStrBase64, cfgPath) - if output, err := exec.Command("/bin/sh", "-c", cmd).Output(); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("Failed to write config to %s: %s", cfgPath, strings.TrimSpace(string(output)))) - } - configPath = cfgPath - } else { - currentDir, err := filepath.Abs(filepath.Dir(os.Args[0])) - if err != nil { - return errors.Wrap(err, "Failed to get current dir") - } - configPath = fmt.Sprintf("%s/%s.yaml", currentDir, opt.Name) - CheckConfigFileStatus(configPath) - cmd := fmt.Sprintf("echo %s | base64 -d > %s/%s.yaml", ClusterCfgStrBase64, currentDir, opt.Name) - err1 := exec.Command("/bin/sh", "-c", cmd).Run() - if err1 != nil { - return err1 - } - - } - notice := "Notice: " + fmt.Sprintf("%s has been created. Some parameters need to be filled in by yourself, please complete it.", configPath) - fmt.Printf("\033[1;36m%s\033[0m\n\n", notice) - return nil -} diff --git a/cmd/kk/pkg/config/generate.go b/cmd/kk/pkg/config/generate.go deleted file mode 100644 index 91a49fcb..00000000 --- a/cmd/kk/pkg/config/generate.go +++ /dev/null @@ -1,137 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package config - -import ( - "bufio" - "encoding/base64" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/pkg/errors" - versionutil "k8s.io/apimachinery/pkg/util/version" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/config/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -// GenerateKubeKeyConfig is used to generate cluster configuration file -func GenerateKubeKeyConfig(arg common.Argument, name string) error { - if arg.FromCluster { - err := GenerateConfigFromCluster(arg.FilePath, arg.KubeConfig, name) - if err != nil { - return err - } - return nil - } - - opt := new(templates.Options) - - if name != "" { - output := strings.Split(name, ".") - opt.Name = output[0] - } else { - opt.Name = "sample" - } - if len(arg.KubernetesVersion) == 0 { - opt.KubeVersion = kubekeyapiv1alpha2.DefaultKubeVersion - } else { - opt.KubeVersion = arg.KubernetesVersion - } - - if k8sVersion, err := versionutil.ParseGeneric(opt.KubeVersion); err == nil { - if k8sVersion.AtLeast(versionutil.MustParseSemantic("v1.24.0")) { - opt.ContainerManager = common.Containerd - } else { - opt.ContainerManager = common.Docker - } - } - - opt.KubeSphereEnabled = arg.KsEnable - - if arg.KsEnable { - version := strings.TrimSpace(arg.KsVersion) - ksInstaller, ok := kubesphere.StabledVersionSupport(version) - if ok { - opt.KubeSphereConfigMap = ksInstaller.CCToString() - } else if latest, ok := kubesphere.LatestRelease(version); ok { - latest.Version = version - opt.KubeSphereConfigMap = latest.CCToString() - } else if dev, ok := kubesphere.DevRelease(version); ok { - dev.Version = version - opt.KubeSphereConfigMap = dev.CCToString() - } else { - return errors.New(fmt.Sprintf("Unsupported KubeSphere version: %s", version)) - } - } - - ClusterObjStr, err := templates.GenerateCluster(opt) - if err != nil { - return errors.Wrap(err, "Failed to generate cluster config") - } - ClusterObjStrBase64 := base64.StdEncoding.EncodeToString([]byte(ClusterObjStr)) - - if arg.FilePath != "" { - CheckConfigFileStatus(arg.FilePath) - cmdStr := fmt.Sprintf("echo %s | base64 -d > %s", ClusterObjStrBase64, arg.FilePath) - output, err := exec.Command("/bin/sh", "-c", cmdStr).CombinedOutput() - if err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("Failed to write config to %s: %s", arg.FilePath, strings.TrimSpace(string(output)))) - } - } else { - currentDir, err := filepath.Abs(filepath.Dir(os.Args[0])) - if err != nil { - return errors.Wrap(err, "Failed to get current dir") - } - CheckConfigFileStatus(fmt.Sprintf("%s/config-%s.yaml", currentDir, opt.Name)) - cmd := fmt.Sprintf("echo %s | base64 -d > %s/config-%s.yaml", ClusterObjStrBase64, currentDir, opt.Name) - if err := exec.Command("/bin/sh", "-c", cmd).Run(); err != nil { - return err - } - } - - fmt.Println("Generate KubeKey config file successfully") - return nil -} - -// CheckConfigFileStatus is used to check the status of cluster configuration file. -func CheckConfigFileStatus(path string) { - if util.IsExist(path) { - reader := bufio.NewReader(os.Stdin) - Loop: - for { - fmt.Printf("%s already exists. Are you sure you want to overwrite this config file? [yes/no]: ", path) - input, _ := reader.ReadString('\n') - input = strings.TrimSpace(input) - - if input != "" { - switch input { - case "yes": - break Loop - case "no": - os.Exit(0) - } - } - } - } -} diff --git a/cmd/kk/pkg/config/templates/cluster.go b/cmd/kk/pkg/config/templates/cluster.go deleted file mode 100644 index 373d4bd4..00000000 --- a/cmd/kk/pkg/config/templates/cluster.go +++ /dev/null @@ -1,93 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -// Cluster defines the template of cluster configuration file default. -var Cluster = template.Must(template.New("Cluster").Parse( - dedent.Dedent(` -apiVersion: kubekey.kubesphere.io/v1alpha2 -kind: Cluster -metadata: - name: {{ .Options.Name }} -spec: - hosts: - - {name: node1, address: 172.16.0.2, internalAddress: 172.16.0.2, user: ubuntu, password: "Qcloud@123"} - - {name: node2, address: 172.16.0.3, internalAddress: 172.16.0.3, user: ubuntu, password: "Qcloud@123"} - roleGroups: - etcd: - - node1 - control-plane: - - node1 - worker: - - node1 - - node2 - controlPlaneEndpoint: - ## Internal loadbalancer for apiservers - # internalLoadbalancer: haproxy - - domain: lb.kubesphere.local - address: "" - port: 6443 - kubernetes: - version: {{ .Options.KubeVersion }} - clusterName: cluster.local - autoRenewCerts: true - containerManager: {{ .Options.ContainerManager }} - etcd: - type: kubekey - network: - plugin: calico - kubePodsCIDR: 10.233.64.0/18 - kubeServiceCIDR: 10.233.0.0/18 - ## multus support. https://github.com/k8snetworkplumbingwg/multus-cni - multusCNI: - enabled: false - registry: - privateRegistry: "" - namespaceOverride: "" - registryMirrors: [] - insecureRegistries: [] - addons: [] - -{{ if .Options.KubeSphereEnabled }} -{{ .Options.KubeSphereConfigMap }} -{{ end }} - `))) - -// Options defines the parameters of cluster configuration. -type Options struct { - Name string - KubeVersion string - KubeSphereEnabled bool - KubeSphereConfigMap string - ContainerManager string -} - -// GenerateCluster is used to generate cluster configuration content. -func GenerateCluster(opt *Options) (string, error) { - return util.Render(Cluster, util.Data{ - "Options": opt, - }) -} diff --git a/cmd/kk/pkg/container/common.go b/cmd/kk/pkg/container/common.go deleted file mode 100644 index 27e8d057..00000000 --- a/cmd/kk/pkg/container/common.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package container - -const ( - DefaultContainerdCRISocket = "/run/containerd/containerd.sock" -) diff --git a/cmd/kk/pkg/container/containerd.go b/cmd/kk/pkg/container/containerd.go deleted file mode 100644 index 05caae1e..00000000 --- a/cmd/kk/pkg/container/containerd.go +++ /dev/null @@ -1,530 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package container - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -type SyncContainerd struct { - common.KubeAction -} - -func (s *SyncContainerd) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - containerd, ok := binariesMap[common.Containerd] - if !ok { - return errors.New("get KubeBinary key containerd by pipeline cache failed") - } - - dst := filepath.Join(common.TmpDir, containerd.FileName) - if err := runtime.GetRunner().Scp(containerd.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync containerd binaries failed")) - } - - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("mkdir -p /usr/bin && tar -zxf %s && mv bin/* /usr/bin && rm -rf bin", dst), - false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("install containerd binaries failed")) - } - return nil -} - -type SyncCrictlBinaries struct { - common.KubeAction -} - -func (s *SyncCrictlBinaries) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - crictl, ok := binariesMap[common.Crictl] - if !ok { - return errors.New("get KubeBinary key crictl by pipeline cache failed") - } - - dst := filepath.Join(common.TmpDir, crictl.FileName) - - if err := runtime.GetRunner().Scp(crictl.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync crictl binaries failed")) - } - - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("mkdir -p /usr/bin && tar -zxf %s -C /usr/bin ", dst), - false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("install crictl binaries failed")) - } - return nil -} - -type EnableContainerd struct { - common.KubeAction -} - -func (e *EnableContainerd) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - "systemctl daemon-reload && systemctl enable containerd && systemctl start containerd", - false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("enable and start containerd failed")) - } - - // install runc - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binariesMapObj, ok := e.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - containerd, ok := binariesMap[common.Runc] - if !ok { - return errors.New("get KubeBinary key runc by pipeline cache failed") - } - - dst := filepath.Join(common.TmpDir, containerd.FileName) - if err := runtime.GetRunner().Scp(containerd.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync runc binaries failed")) - } - - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("install -m 755 %s /usr/local/sbin/runc", dst), - false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("install runc binaries failed")) - } - return nil -} - -type DisableContainerd struct { - common.KubeAction -} - -func (d *DisableContainerd) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - "systemctl disable containerd && systemctl stop containerd", true); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("disable and stop containerd failed")) - } - - // remove containerd related files - files := []string{ - "/usr/local/sbin/runc", - "/usr/bin/crictl", - "/usr/bin/containerd*", - "/usr/bin/ctr", - filepath.Join("/etc/systemd/system", templates.ContainerdService.Name()), - filepath.Join("/etc/containerd", templates.ContainerdConfig.Name()), - filepath.Join("/etc", templates.CrictlConfig.Name()), - } - if d.KubeConf.Cluster.Registry.DataRoot != "" { - files = append(files, d.KubeConf.Cluster.Registry.DataRoot) - } else { - files = append(files, "/var/lib/containerd") - } - - for _, file := range files { - _, _ = runtime.GetRunner().SudoCmd(fmt.Sprintf("rm -rf %s", file), true) - } - return nil -} - -type CordonNode struct { - common.KubeAction -} - -func (d *CordonNode) Execute(runtime connector.Runtime) error { - nodeName := runtime.RemoteHost().GetName() - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("/usr/local/bin/kubectl cordon %s ", nodeName), true); err != nil { - return errors.Wrap(err, fmt.Sprintf("cordon the node: %s failed", nodeName)) - } - return nil -} - -type UnCordonNode struct { - common.KubeAction -} - -func (d *UnCordonNode) Execute(runtime connector.Runtime) error { - nodeName := runtime.RemoteHost().GetName() - f := true - for f { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("/usr/local/bin/kubectl uncordon %s", nodeName), true); err == nil { - break - } - - } - return nil -} - -type DrainNode struct { - common.KubeAction -} - -func (d *DrainNode) Execute(runtime connector.Runtime) error { - nodeName := runtime.RemoteHost().GetName() - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("/usr/local/bin/kubectl drain %s --delete-emptydir-data --ignore-daemonsets --timeout=2m --force", nodeName), true); err != nil { - return errors.Wrap(err, fmt.Sprintf("drain the node: %s failed", nodeName)) - } - return nil -} - -type RestartCri struct { - common.KubeAction -} - -func (i *RestartCri) Execute(runtime connector.Runtime) error { - switch i.KubeConf.Arg.Type { - case common.Docker: - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("systemctl daemon-reload && systemctl restart docker "), true); err != nil { - return errors.Wrap(err, "restart docker") - } - case common.Containerd: - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("systemctl daemon-reload && systemctl restart containerd"), true); err != nil { - return errors.Wrap(err, "restart containerd") - } - - default: - logger.Log.Fatalf("Unsupported container runtime: %s", strings.TrimSpace(i.KubeConf.Arg.Type)) - } - return nil -} - -type EditKubeletCri struct { - common.KubeAction -} - -func (i *EditKubeletCri) Execute(runtime connector.Runtime) error { - switch i.KubeConf.Arg.Type { - case common.Docker: - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "sed -i 's#--container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --pod#--pod#' /var/lib/kubelet/kubeadm-flags.env"), - true); err != nil { - return errors.Wrap(err, "Change KubeletTo Containerd failed") - } - case common.Containerd: - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "sed -i 's#--network-plugin=cni --pod#--network-plugin=cni --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --pod#' /var/lib/kubelet/kubeadm-flags.env"), - true); err != nil { - return errors.Wrap(err, "Change KubeletTo Containerd failed") - } - - default: - logger.Log.Fatalf("Unsupported container runtime: %s", strings.TrimSpace(i.KubeConf.Arg.Type)) - } - return nil -} - -type RestartKubeletNode struct { - common.KubeAction -} - -func (d *RestartKubeletNode) Execute(runtime connector.Runtime) error { - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("systemctl restart kubelet"), true); err != nil { - return errors.Wrap(err, "RestartNode Kube failed") - } - return nil -} - -func MigrateSelfNodeCriTasks(runtime connector.Runtime, kubeAction common.KubeAction) error { - host := runtime.RemoteHost() - tasks := []task.Interface{} - CordonNode := &task.RemoteTask{ - Name: "CordonNode", - Desc: "Cordon Node", - Hosts: []connector.Host{host}, - - Action: new(CordonNode), - Parallel: false, - } - DrainNode := &task.RemoteTask{ - Name: "DrainNode", - Desc: "Drain Node", - Hosts: []connector.Host{host}, - Action: new(DrainNode), - Parallel: false, - } - RestartCri := &task.RemoteTask{ - Name: "RestartCri", - Desc: "Restart Cri", - Hosts: []connector.Host{host}, - Action: new(RestartCri), - Parallel: false, - } - EditKubeletCri := &task.RemoteTask{ - Name: "EditKubeletCri", - Desc: "Edit Kubelet Cri", - Hosts: []connector.Host{host}, - Action: new(EditKubeletCri), - Parallel: false, - } - RestartKubeletNode := &task.RemoteTask{ - Name: "RestartKubeletNode", - Desc: "Restart Kubelet Node", - Hosts: []connector.Host{host}, - Action: new(RestartKubeletNode), - Parallel: false, - } - UnCordonNode := &task.RemoteTask{ - Name: "UnCordonNode", - Desc: "UnCordon Node", - Hosts: []connector.Host{host}, - Action: new(UnCordonNode), - Parallel: false, - } - switch kubeAction.KubeConf.Cluster.Kubernetes.ContainerManager { - case common.Docker: - Uninstall := &task.RemoteTask{ - Name: "DisableDocker", - Desc: "Disable docker", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - &DockerExist{Not: false}, - }, - Action: new(DisableDocker), - Parallel: false, - } - tasks = append(tasks, CordonNode, DrainNode, Uninstall) - case common.Containerd: - Uninstall := &task.RemoteTask{ - Name: "UninstallContainerd", - Desc: "Uninstall containerd", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - &ContainerdExist{Not: false}, - }, - Action: new(DisableContainerd), - Parallel: false, - } - tasks = append(tasks, CordonNode, DrainNode, Uninstall) - } - if kubeAction.KubeConf.Arg.Type == common.Docker { - syncBinaries := &task.RemoteTask{ - Name: "SyncDockerBinaries", - Desc: "Sync docker binaries", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - // &kubernetes.NodeInCluster{Not: true}, - &DockerExist{Not: true}, - }, - Action: new(SyncDockerBinaries), - Parallel: false, - } - generateDockerService := &task.RemoteTask{ - Name: "GenerateDockerService", - Desc: "Generate docker service", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - // &kubernetes.NodeInCluster{Not: true}, - &DockerExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.DockerService, - Dst: filepath.Join("/etc/systemd/system", templates.DockerService.Name()), - }, - Parallel: false, - } - generateDockerConfig := &task.RemoteTask{ - Name: "GenerateDockerConfig", - Desc: "Generate docker config", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - // &kubernetes.NodeInCluster{Not: true}, - &DockerExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.DockerConfig, - Dst: filepath.Join("/etc/docker/", templates.DockerConfig.Name()), - Data: util.Data{ - "Mirrors": templates.Mirrors(kubeAction.KubeConf), - "InsecureRegistries": templates.InsecureRegistries(kubeAction.KubeConf), - "DataRoot": templates.DataRoot(kubeAction.KubeConf), - }, - }, - Parallel: false, - } - enableDocker := &task.RemoteTask{ - Name: "EnableDocker", - Desc: "Enable docker", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - // &kubernetes.NodeInCluster{Not: true}, - &DockerExist{Not: true}, - }, - Action: new(EnableDocker), - Parallel: false, - } - dockerLoginRegistry := &task.RemoteTask{ - Name: "Login PrivateRegistry", - Desc: "Add auths to container runtime", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - // &kubernetes.NodeInCluster{Not: true}, - &DockerExist{}, - &PrivateRegistryAuth{}, - }, - Action: new(DockerLoginRegistry), - Parallel: false, - } - - tasks = append(tasks, syncBinaries, generateDockerService, generateDockerConfig, enableDocker, dockerLoginRegistry, - RestartCri, EditKubeletCri, RestartKubeletNode, UnCordonNode) - } - if kubeAction.KubeConf.Arg.Type == common.Containerd { - syncContainerd := &task.RemoteTask{ - Name: "SyncContainerd", - Desc: "Sync containerd binaries", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - &ContainerdExist{Not: true}, - }, - Action: new(SyncContainerd), - Parallel: false, - } - - syncCrictlBinaries := &task.RemoteTask{ - Name: "SyncCrictlBinaries", - Desc: "Sync crictl binaries", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - &CrictlExist{Not: true}, - }, - Action: new(SyncCrictlBinaries), - Parallel: false, - } - - generateContainerdService := &task.RemoteTask{ - Name: "GenerateContainerdService", - Desc: "Generate containerd service", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - &ContainerdExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.ContainerdService, - Dst: filepath.Join("/etc/systemd/system", templates.ContainerdService.Name()), - }, - Parallel: false, - } - - generateContainerdConfig := &task.RemoteTask{ - Name: "GenerateContainerdConfig", - Desc: "Generate containerd config", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - &ContainerdExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.ContainerdConfig, - Dst: filepath.Join("/etc/containerd/", templates.ContainerdConfig.Name()), - Data: util.Data{ - "Mirrors": templates.Mirrors(kubeAction.KubeConf), - "InsecureRegistries": kubeAction.KubeConf.Cluster.Registry.InsecureRegistries, - "SandBoxImage": images.GetImage(runtime, kubeAction.KubeConf, "pause").ImageName(), - "Auths": registry.DockerRegistryAuthEntries(kubeAction.KubeConf.Cluster.Registry.Auths), - "DataRoot": templates.DataRoot(kubeAction.KubeConf), - }, - }, - Parallel: false, - } - - generateCrictlConfig := &task.RemoteTask{ - Name: "GenerateCrictlConfig", - Desc: "Generate crictl config", - Hosts: []connector.Host{host}, - Prepare: &prepare.PrepareCollection{ - &ContainerdExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.CrictlConfig, - Dst: filepath.Join("/etc/", templates.CrictlConfig.Name()), - Data: util.Data{ - "Endpoint": kubeAction.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, - }, - }, - Parallel: false, - } - - enableContainerd := &task.RemoteTask{ - Name: "EnableContainerd", - Desc: "Enable containerd", - Hosts: []connector.Host{host}, - // Prepare: &prepare.PrepareCollection{ - // &ContainerdExist{Not: true}, - // }, - Action: new(EnableContainerd), - Parallel: false, - } - tasks = append(tasks, syncContainerd, syncCrictlBinaries, generateContainerdService, generateContainerdConfig, - generateCrictlConfig, enableContainerd, RestartCri, EditKubeletCri, RestartKubeletNode, UnCordonNode) - } - - for i := range tasks { - t := tasks[i] - t.Init(runtime, kubeAction.ModuleCache, kubeAction.PipelineCache) - if res := t.Execute(); res.IsFailed() { - return res.CombineErr() - } - } - return nil -} - -type MigrateSelfNodeCri struct { - common.KubeAction -} - -func (d *MigrateSelfNodeCri) Execute(runtime connector.Runtime) error { - - if err := MigrateSelfNodeCriTasks(runtime, d.KubeAction); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("MigrateSelfNodeCriTasks failed:")) - } - return nil -} diff --git a/cmd/kk/pkg/container/docker.go b/cmd/kk/pkg/container/docker.go deleted file mode 100644 index 0beecff3..00000000 --- a/cmd/kk/pkg/container/docker.go +++ /dev/null @@ -1,148 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package container - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -type SyncDockerBinaries struct { - common.KubeAction -} - -func (s *SyncDockerBinaries) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - docker, ok := binariesMap[common.Docker] - if !ok { - return errors.New("get KubeBinary key docker by pipeline cache failed") - } - - dst := filepath.Join(common.TmpDir, docker.FileName) - if err := runtime.GetRunner().Scp(docker.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync docker binaries failed")) - } - - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("mkdir -p /usr/bin && tar -zxf %s && mv docker/* /usr/bin && rm -rf docker", dst), - false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("install container runtime docker binaries failed")) - } - return nil -} - -type EnableDocker struct { - common.KubeAction -} - -func (e *EnableDocker) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - "systemctl daemon-reload && systemctl enable docker && systemctl start docker", - false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("enable and start docker failed")) - } - return nil -} - -type DockerLoginRegistry struct { - common.KubeAction -} - -func (p *DockerLoginRegistry) Execute(runtime connector.Runtime) error { - - auths := registry.DockerRegistryAuthEntries(p.KubeConf.Cluster.Registry.Auths) - - for repo, entry := range auths { - if len(entry.Username) == 0 || len(entry.Password) == 0 { - continue - } - cmd := fmt.Sprintf("HOME=$HOME docker login --username '%s' --password '%s' %s", escapeSpecialCharacters(entry.Username), escapeSpecialCharacters(entry.Password), repo) - if _, err := runtime.GetRunner().SudoCmd(cmd, false); err != nil { - return errors.Wrapf(err, "login registry failed, cmd: %v, err:%v", cmd, err) - } - } - - if output, err := runtime.GetRunner().SudoCmd( - "if [ -e $HOME/.docker/config.json ]; "+ - "then echo 'exist'; "+ - "fi", false); err == nil && strings.Contains(output, "exist") { - - cmd := "mkdir -p /.docker && cp -f $HOME/.docker/config.json /.docker/ && chmod 0644 /.docker/config.json " - if _, err := runtime.GetRunner().SudoCmd(cmd, false); err != nil { - return errors.Wrapf(err, "copy docker auths failed cmd: %v, err:%v", cmd, err) - } - } - - return nil -} - -type DisableDocker struct { - common.KubeAction -} - -func (d *DisableDocker) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl disable docker && systemctl stop docker", - false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("disable and stop docker failed")) - } - - // remove docker related files - files := []string{ - "/usr/bin/runc", - "/usr/bin/ctr", - "/usr/bin/docker*", - "/usr/bin/containerd*", - filepath.Join("/etc/systemd/system", templates.DockerService.Name()), - filepath.Join("/etc/docker", templates.DockerConfig.Name()), - } - if d.KubeConf.Cluster.Registry.DataRoot != "" { - files = append(files, d.KubeConf.Cluster.Registry.DataRoot) - } else { - files = append(files, "/var/lib/docker") - } - - for _, file := range files { - _, _ = runtime.GetRunner().SudoCmd(fmt.Sprintf("rm -rf %s", file), true) - } - return nil -} - -func escapeSpecialCharacters(str string) string { - newStr := strings.ReplaceAll(str, "$", "\\$") - newStr = strings.ReplaceAll(newStr, "&", "\\&") - newStr = strings.ReplaceAll(newStr, "*", "\\*") - return newStr -} diff --git a/cmd/kk/pkg/container/module.go b/cmd/kk/pkg/container/module.go deleted file mode 100644 index 3cdab21a..00000000 --- a/cmd/kk/pkg/container/module.go +++ /dev/null @@ -1,388 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package container - -import ( - "path/filepath" - "strings" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry" -) - -type InstallContainerModule struct { - common.KubeModule - Skip bool -} - -func (i *InstallContainerModule) IsSkip() bool { - return i.Skip -} - -func (i *InstallContainerModule) Init() { - i.Name = "InstallContainerModule" - i.Desc = "Install container manager" - - switch i.KubeConf.Cluster.Kubernetes.ContainerManager { - case common.Docker: - i.Tasks = InstallDocker(i) - case common.Containerd: - i.Tasks = InstallContainerd(i) - case common.Crio: - // TODO: Add the steps of cri-o's installation. - case common.Isula: - // TODO: Add the steps of iSula's installation. - default: - logger.Log.Fatalf("Unsupported container runtime: %s", strings.TrimSpace(i.KubeConf.Cluster.Kubernetes.ContainerManager)) - } -} - -func InstallDocker(m *InstallContainerModule) []task.Interface { - syncBinaries := &task.RemoteTask{ - Name: "SyncDockerBinaries", - Desc: "Sync docker binaries", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &DockerExist{Not: true}, - }, - Action: new(SyncDockerBinaries), - Parallel: true, - Retry: 2, - } - - generateDockerService := &task.RemoteTask{ - Name: "GenerateDockerService", - Desc: "Generate docker service", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &DockerExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.DockerService, - Dst: filepath.Join("/etc/systemd/system", templates.DockerService.Name()), - }, - Parallel: true, - } - - generateDockerConfig := &task.RemoteTask{ - Name: "GenerateDockerConfig", - Desc: "Generate docker config", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &DockerExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.DockerConfig, - Dst: filepath.Join("/etc/docker/", templates.DockerConfig.Name()), - Data: util.Data{ - "Mirrors": templates.Mirrors(m.KubeConf), - "InsecureRegistries": templates.InsecureRegistries(m.KubeConf), - "DataRoot": templates.DataRoot(m.KubeConf), - "BridgeIP": templates.BridgeIP(m.KubeConf), - }, - }, - Parallel: true, - } - - enableDocker := &task.RemoteTask{ - Name: "EnableDocker", - Desc: "Enable docker", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &DockerExist{Not: true}, - }, - Action: new(EnableDocker), - Parallel: true, - } - - dockerLoginRegistry := &task.RemoteTask{ - Name: "Login PrivateRegistry", - Desc: "Add auths to container runtime", - Hosts: m.Runtime.GetAllHosts(), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &DockerExist{}, - &PrivateRegistryAuth{}, - }, - Action: new(DockerLoginRegistry), - Parallel: true, - } - - return []task.Interface{ - syncBinaries, - generateDockerService, - generateDockerConfig, - enableDocker, - dockerLoginRegistry, - } -} - -func InstallContainerd(m *InstallContainerModule) []task.Interface { - syncContainerd := &task.RemoteTask{ - Name: "SyncContainerd", - Desc: "Sync containerd binaries", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &ContainerdExist{Not: true}, - }, - Action: new(SyncContainerd), - Parallel: true, - Retry: 2, - } - - syncCrictlBinaries := &task.RemoteTask{ - Name: "SyncCrictlBinaries", - Desc: "Sync crictl binaries", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &CrictlExist{Not: true}, - }, - Action: new(SyncCrictlBinaries), - Parallel: true, - Retry: 2, - } - - generateContainerdService := &task.RemoteTask{ - Name: "GenerateContainerdService", - Desc: "Generate containerd service", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &ContainerdExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.ContainerdService, - Dst: filepath.Join("/etc/systemd/system", templates.ContainerdService.Name()), - }, - Parallel: true, - } - - generateContainerdConfig := &task.RemoteTask{ - Name: "GenerateContainerdConfig", - Desc: "Generate containerd config", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &ContainerdExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.ContainerdConfig, - Dst: filepath.Join("/etc/containerd/", templates.ContainerdConfig.Name()), - Data: util.Data{ - "Mirrors": templates.Mirrors(m.KubeConf), - "InsecureRegistries": m.KubeConf.Cluster.Registry.InsecureRegistries, - "SandBoxImage": images.GetImage(m.Runtime, m.KubeConf, "pause").ImageName(), - "Auths": registry.DockerRegistryAuthEntries(m.KubeConf.Cluster.Registry.Auths), - "DataRoot": templates.DataRoot(m.KubeConf), - }, - }, - Parallel: true, - } - - generateCrictlConfig := &task.RemoteTask{ - Name: "GenerateCrictlConfig", - Desc: "Generate crictl config", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &ContainerdExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.CrictlConfig, - Dst: filepath.Join("/etc/", templates.CrictlConfig.Name()), - Data: util.Data{ - "Endpoint": m.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, - }, - }, - Parallel: true, - } - - enableContainerd := &task.RemoteTask{ - Name: "EnableContainerd", - Desc: "Enable containerd", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &kubernetes.NodeInCluster{Not: true}, - &ContainerdExist{Not: true}, - }, - Action: new(EnableContainerd), - Parallel: true, - } - - return []task.Interface{ - syncContainerd, - syncCrictlBinaries, - generateContainerdService, - generateContainerdConfig, - generateCrictlConfig, - enableContainerd, - } -} - -type UninstallContainerModule struct { - common.KubeModule - Skip bool -} - -func (i *UninstallContainerModule) IsSkip() bool { - return i.Skip -} - -func (i *UninstallContainerModule) Init() { - i.Name = "UninstallContainerModule" - i.Desc = "Uninstall container manager" - - switch i.KubeConf.Cluster.Kubernetes.ContainerManager { - case common.Docker: - i.Tasks = UninstallDocker(i) - case common.Containerd: - i.Tasks = UninstallContainerd(i) - case common.Crio: - // TODO: Add the steps of cri-o's installation. - case common.Isula: - // TODO: Add the steps of iSula's installation. - default: - logger.Log.Fatalf("Unsupported container runtime: %s", strings.TrimSpace(i.KubeConf.Cluster.Kubernetes.ContainerManager)) - } -} - -func UninstallDocker(m *UninstallContainerModule) []task.Interface { - - disableDocker := &task.RemoteTask{ - Name: "DisableDocker", - Desc: "Disable docker", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &DockerExist{Not: false}, - }, - Action: new(DisableDocker), - Parallel: true, - } - - return []task.Interface{ - disableDocker, - } -} - -func UninstallContainerd(m *UninstallContainerModule) []task.Interface { - disableContainerd := &task.RemoteTask{ - Name: "UninstallContainerd", - Desc: "Uninstall containerd", - Hosts: m.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &ContainerdExist{Not: false}, - }, - Action: new(DisableContainerd), - Parallel: true, - } - - return []task.Interface{ - disableContainerd, - } -} - -type CriMigrateModule struct { - common.KubeModule - - Skip bool -} - -func (i *CriMigrateModule) IsSkip() bool { - return i.Skip -} - -func (p *CriMigrateModule) Init() { - p.Name = "CriMigrateModule" - p.Desc = "Cri Migrate manager" - - if p.KubeConf.Arg.Role == common.Worker { - p.Tasks = MigrateWCri(p) - } else if p.KubeConf.Arg.Role == common.Master { - p.Tasks = MigrateMCri(p) - } else if p.KubeConf.Arg.Role == "all" { - p.Tasks = MigrateACri(p) - } else { - logger.Log.Fatalf("Unsupported Role: %s", strings.TrimSpace(p.KubeConf.Arg.Role)) - } -} - -func MigrateWCri(p *CriMigrateModule) []task.Interface { - - MigrateWCri := &task.RemoteTask{ - Name: "MigrateToDocker", - Desc: "Migrate To Docker", - Hosts: p.Runtime.GetHostsByRole(common.Worker), - Prepare: new(common.OnlyWorker), - Action: new(MigrateSelfNodeCri), - Parallel: false, - } - - p.Tasks = []task.Interface{ - MigrateWCri, - } - - return p.Tasks -} - -func MigrateMCri(p *CriMigrateModule) []task.Interface { - - MigrateMCri := &task.RemoteTask{ - Name: "MigrateMasterToDocker", - Desc: "Migrate Master To Docker", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.IsMaster), - Action: new(MigrateSelfNodeCri), - Parallel: false, - } - - p.Tasks = []task.Interface{ - MigrateMCri, - } - - return p.Tasks -} - -func MigrateACri(p *CriMigrateModule) []task.Interface { - - MigrateACri := &task.RemoteTask{ - Name: "MigrateMasterToDocker", - Desc: "Migrate Master To Docker", - Hosts: p.Runtime.GetHostsByRole(common.K8s), - Action: new(MigrateSelfNodeCri), - Parallel: false, - } - - p.Tasks = []task.Interface{ - MigrateACri, - } - - return p.Tasks -} diff --git a/cmd/kk/pkg/container/prepares.go b/cmd/kk/pkg/container/prepares.go deleted file mode 100644 index 34237857..00000000 --- a/cmd/kk/pkg/container/prepares.go +++ /dev/null @@ -1,92 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package container - -import ( - "strings" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type DockerExist struct { - common.KubePrepare - Not bool -} - -func (d *DockerExist) PreCheck(runtime connector.Runtime) (bool, error) { - output, err := runtime.GetRunner().SudoCmd("if [ -z $(which docker) ] || [ ! -e /var/run/docker.sock ]; "+ - "then echo 'not exist'; "+ - "fi", false) - if err != nil { - return false, err - } - if strings.Contains(output, "not exist") { - return d.Not, nil - } - return !d.Not, nil -} - -type CrictlExist struct { - common.KubePrepare - Not bool -} - -func (c *CrictlExist) PreCheck(runtime connector.Runtime) (bool, error) { - output, err := runtime.GetRunner().SudoCmd( - "if [ -z $(which crictl) ]; "+ - "then echo 'not exist'; "+ - "fi", false) - if err != nil { - return false, err - } - if strings.Contains(output, "not exist") { - return c.Not, nil - } else { - return !c.Not, nil - } -} - -type ContainerdExist struct { - common.KubePrepare - Not bool -} - -func (c *ContainerdExist) PreCheck(runtime connector.Runtime) (bool, error) { - output, err := runtime.GetRunner().SudoCmd( - "if [ -z $(which containerd) ] || [ ! -e /run/containerd/containerd.sock ]; "+ - "then echo 'not exist'; "+ - "fi", false) - if err != nil { - return false, err - } - if strings.Contains(output, "not exist") { - return c.Not, nil - } - return !c.Not, nil -} - -type PrivateRegistryAuth struct { - common.KubePrepare -} - -func (p *PrivateRegistryAuth) PreCheck(runtime connector.Runtime) (bool, error) { - if len(p.KubeConf.Cluster.Registry.Auths.Raw) == 0 { - return false, nil - } - return true, nil -} diff --git a/cmd/kk/pkg/container/templates/containerd_config.go b/cmd/kk/pkg/container/templates/containerd_config.go deleted file mode 100644 index 1d2739ba..00000000 --- a/cmd/kk/pkg/container/templates/containerd_config.go +++ /dev/null @@ -1,104 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var ContainerdConfig = template.Must(template.New("config.toml").Parse( - dedent.Dedent(`version = 2 -{{- if .DataRoot }} -root = {{ .DataRoot }} -{{ else }} -root = "/var/lib/containerd" -{{- end }} -state = "/run/containerd" - -[grpc] - address = "/run/containerd/containerd.sock" - uid = 0 - gid = 0 - max_recv_message_size = 16777216 - max_send_message_size = 16777216 - -[ttrpc] - address = "" - uid = 0 - gid = 0 - -[debug] - address = "" - uid = 0 - gid = 0 - level = "" - -[metrics] - address = "" - grpc_histogram = false - -[cgroup] - path = "" - -[timeouts] - "io.containerd.timeout.shim.cleanup" = "5s" - "io.containerd.timeout.shim.load" = "5s" - "io.containerd.timeout.shim.shutdown" = "3s" - "io.containerd.timeout.task.state" = "2s" - -[plugins] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri"] - sandbox_image = "{{ .SandBoxImage }}" - [plugins."io.containerd.grpc.v1.cri".cni] - bin_dir = "/opt/cni/bin" - conf_dir = "/etc/cni/net.d" - max_conf_num = 1 - conf_template = "" - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - {{- if .Mirrors }} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = [{{ .Mirrors }}, "https://registry-1.docker.io"] - {{ else }} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - {{- end}} - {{- range $value := .InsecureRegistries }} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$value}}"] - endpoint = ["http://{{$value}}"] - {{- end}} - - {{- if .Auths }} - [plugins."io.containerd.grpc.v1.cri".registry.configs] - {{- range $repo, $entry := .Auths }} - [plugins."io.containerd.grpc.v1.cri".registry.configs."{{$repo}}".auth] - username = "{{$entry.Username}}" - password = "{{$entry.Password}}" - [plugins."io.containerd.grpc.v1.cri".registry.configs."{{$repo}}".tls] - ca_file = "{{$entry.CAFile}}" - cert_file = "{{$entry.CertFile}}" - key_file = "{{$entry.KeyFile}}" - insecure_skip_verify = {{$entry.SkipTLSVerify}} - {{- end}} - {{- end}} - `))) diff --git a/cmd/kk/pkg/container/templates/containerd_service.go b/cmd/kk/pkg/container/templates/containerd_service.go deleted file mode 100644 index 366ff73e..00000000 --- a/cmd/kk/pkg/container/templates/containerd_service.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var ContainerdService = template.Must(template.New("containerd.service").Parse( - dedent.Dedent(`[Unit] -Description=containerd container runtime -Documentation=https://containerd.io -After=network.target local-fs.target - -[Service] -ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/bin/containerd - -Type=notify -Delegate=yes -KillMode=process -Restart=always -RestartSec=5 -# Having non-zero Limit*s causes performance problems due to accounting overhead -# in the kernel. We recommend using cgroups to do container-local accounting. -LimitNPROC=infinity -LimitCORE=infinity -LimitNOFILE=1048576 -# Comment TasksMax if your systemd version does not supports it. -# Only systemd 226 and above support this version. -TasksMax=infinity -OOMScoreAdjust=-999 - -[Install] -WantedBy=multi-user.target - `))) diff --git a/cmd/kk/pkg/container/templates/crictl_config.go b/cmd/kk/pkg/container/templates/crictl_config.go deleted file mode 100644 index 885d60ae..00000000 --- a/cmd/kk/pkg/container/templates/crictl_config.go +++ /dev/null @@ -1,30 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var CrictlConfig = template.Must(template.New("crictl.yaml").Parse( - dedent.Dedent(`runtime-endpoint: {{ .Endpoint }} -image-endpoint: {{ .Endpoint }} -timeout: 5 -debug: false -pull-image-on-create: false - `))) diff --git a/cmd/kk/pkg/container/templates/docker_config.go b/cmd/kk/pkg/container/templates/docker_config.go deleted file mode 100644 index 069ea979..00000000 --- a/cmd/kk/pkg/container/templates/docker_config.go +++ /dev/null @@ -1,106 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "fmt" - "net" - "strings" - "text/template" - - "github.com/lithammer/dedent" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" -) - -var DockerConfig = template.Must(template.New("daemon.json").Parse( - dedent.Dedent(`{ - "log-opts": { - "max-size": "5m", - "max-file":"3" - }, - {{- if .DataRoot }} - "data-root": {{ .DataRoot }}, - {{- end}} - {{- if .Mirrors }} - "registry-mirrors": [{{ .Mirrors }}], - {{- end}} - {{- if .InsecureRegistries }} - "insecure-registries": [{{ .InsecureRegistries }}], - {{- end}} - {{- if .BridgeIP }} - "bip": {{ .BridgeIP }}, - {{- end}} - "exec-opts": ["native.cgroupdriver=systemd"] -} - `))) - -func Mirrors(kubeConf *common.KubeConf) string { - var mirrors string - if kubeConf.Cluster.Registry.RegistryMirrors != nil { - var mirrorsArr []string - for _, mirror := range kubeConf.Cluster.Registry.RegistryMirrors { - mirrorsArr = append(mirrorsArr, fmt.Sprintf("\"%s\"", mirror)) - } - mirrors = strings.Join(mirrorsArr, ", ") - } - return mirrors -} - -func InsecureRegistries(kubeConf *common.KubeConf) string { - var insecureRegistries string - if kubeConf.Cluster.Registry.InsecureRegistries != nil { - var registriesArr []string - for _, repo := range kubeConf.Cluster.Registry.InsecureRegistries { - registriesArr = append(registriesArr, fmt.Sprintf("\"%s\"", repo)) - } - insecureRegistries = strings.Join(registriesArr, ", ") - } - return insecureRegistries -} - -func DataRoot(kubeConf *common.KubeConf) string { - var dataRoot string - if kubeConf.Cluster.Registry.DataRoot != "" { - dataRoot = fmt.Sprintf("\"%s\"", kubeConf.Cluster.Registry.DataRoot) - } - return dataRoot -} - -func BridgeIP(kubeConf *common.KubeConf) string { - var bip string - if kubeConf.Cluster.Registry.BridgeIP != "" { - bip = "172.17.0.1/16" - _, cidr, err := net.ParseCIDR(kubeConf.Cluster.Registry.BridgeIP) - if err != nil { - return bip - } - ip4 := cidr.IP.To4() - if ip4 == nil { - return bip - } - bridge0 := net.IPv4(ip4[0], ip4[1], ip4[2], ip4[3]+1) - if !cidr.Contains(bridge0) { - return bip - } - ones, _ := cidr.Mask.Size() - - bip = fmt.Sprintf("\"%s/%d\"", bridge0.String(), ones) - } - - return bip -} diff --git a/cmd/kk/pkg/container/templates/docker_service.go b/cmd/kk/pkg/container/templates/docker_service.go deleted file mode 100644 index 55dfc7dd..00000000 --- a/cmd/kk/pkg/container/templates/docker_service.go +++ /dev/null @@ -1,73 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var DockerService = template.Must(template.New("docker.service").Parse( - dedent.Dedent(`[Unit] -Description=Docker Application Container Engine -Documentation=https://docs.docker.com -# After=network-online.target firewalld.service containerd.service -# Wants=network-online.target -# Requires=docker.socket containerd.service - -[Service] -Type=notify -# the default is not to use systemd for cgroups because the delegate issues still -# exists and systemd currently does not support the cgroup feature set required -# for containers run by docker -ExecStart=/usr/bin/dockerd -ExecReload=/bin/kill -s HUP $MAINPID -TimeoutSec=0 -RestartSec=2 -Restart=always - -# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. -# Both the old, and new location are accepted by systemd 229 and up, so using the old location -# to make them work for either version of systemd. -StartLimitBurst=3 - -# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. -# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make -# this option work for either version of systemd. -StartLimitInterval=60s - -# Having non-zero Limit*s causes performance problems due to accounting overhead -# in the kernel. We recommend using cgroups to do container-local accounting. -LimitNOFILE=infinity -LimitNPROC=infinity -LimitCORE=infinity - -# Comment TasksMax if your systemd version does not support it. -# Only systemd 226 and above support this option. -TasksMax=infinity - -# set delegate yes so that systemd does not reset the cgroups of docker containers -Delegate=yes - -# kill only the docker process, not all processes in the cgroup -KillMode=process -OOMScoreAdjust=-500 - -[Install] -WantedBy=multi-user.target - - `))) diff --git a/cmd/kk/pkg/core/action/base.go b/cmd/kk/pkg/core/action/base.go deleted file mode 100644 index 0d09b549..00000000 --- a/cmd/kk/pkg/core/action/base.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package action - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type BaseAction struct { - ModuleCache *cache.Cache - PipelineCache *cache.Cache -} - -func (b *BaseAction) Init(moduleCache *cache.Cache, pipelineCache *cache.Cache) { - b.ModuleCache = moduleCache - b.PipelineCache = pipelineCache -} - -func (b *BaseAction) Execute(runtime connector.Runtime) error { - return nil -} - -func (b *BaseAction) AutoAssert(runtime connector.Runtime) { - -} diff --git a/cmd/kk/pkg/core/action/interface.go b/cmd/kk/pkg/core/action/interface.go deleted file mode 100644 index cc1fd45f..00000000 --- a/cmd/kk/pkg/core/action/interface.go +++ /dev/null @@ -1,28 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package action - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type Action interface { - Execute(runtime connector.Runtime) (err error) - Init(cache *cache.Cache, rootCache *cache.Cache) - AutoAssert(runtime connector.Runtime) -} diff --git a/cmd/kk/pkg/core/action/template.go b/cmd/kk/pkg/core/action/template.go deleted file mode 100644 index cde5d1d2..00000000 --- a/cmd/kk/pkg/core/action/template.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package action - -import ( - "fmt" - "path/filepath" - "text/template" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type Template struct { - BaseAction - Template *template.Template - Dst string - Data util.Data -} - -func (t *Template) Execute(runtime connector.Runtime) error { - templateStr, err := util.Render(t.Template, t.Data) - if err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("render template %s failed", t.Template.Name())) - } - - fileName := filepath.Join(runtime.GetHostWorkDir(), t.Template.Name()) - if err := util.WriteFile(fileName, []byte(templateStr)); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("write file %s failed", fileName)) - } - - if err := runtime.GetRunner().SudoScp(fileName, t.Dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("scp file %s to remote %s failed", fileName, t.Dst)) - } - - return nil -} diff --git a/cmd/kk/pkg/core/cache/cache.go b/cmd/kk/pkg/core/cache/cache.go deleted file mode 100644 index fc5d20a7..00000000 --- a/cmd/kk/pkg/core/cache/cache.go +++ /dev/null @@ -1,85 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cache - -import "sync" - -type Cache struct { - store sync.Map -} - -func NewCache() *Cache { - var m Cache - return &m -} - -func (c *Cache) Set(k string, v interface{}) { - c.store.Store(k, v) -} - -// GetOrSet returns the existing value for the key if present. -// Otherwise, it stores and returns the given value. -// The loaded result is true if the value was loaded, false if stored. -func (c *Cache) GetOrSet(k string, v interface{}) (interface{}, bool) { - return c.store.LoadOrStore(k, v) -} - -func (c *Cache) Get(k string) (interface{}, bool) { - return c.store.Load(k) -} - -func (c *Cache) Range(f func(key, value interface{}) bool) { - c.store.Range(f) -} - -func (c *Cache) Delete(k string) { - c.store.Delete(k) -} - -func (c *Cache) Clean() { - c.store.Range(func(key, value interface{}) bool { - c.store.Delete(key) - return true - }) -} - -func (c *Cache) GetMustInt(k string) (int, bool) { - v, ok := c.Get(k) - res, assert := v.(int) - if !assert { - return res, false - } - return res, ok -} - -func (c *Cache) GetMustString(k string) (string, bool) { - v, ok := c.Get(k) - res, assert := v.(string) - if !assert { - return res, false - } - return res, ok -} - -func (c *Cache) GetMustBool(k string) (bool, bool) { - v, ok := c.Get(k) - res, assert := v.(bool) - if !assert { - return res, false - } - return res, ok -} diff --git a/cmd/kk/pkg/core/common/common.go b/cmd/kk/pkg/core/common/common.go deleted file mode 100644 index 5d42ec45..00000000 --- a/cmd/kk/pkg/core/common/common.go +++ /dev/null @@ -1,37 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package common - -const ( - KubeKey = "kubekey" - - Pipeline = "Pipeline" - Module = "Module" - Task = "Task" - Node = "Node" - - LocalHost = "LocalHost" - - FileMode0755 = 0755 - FileMode0644 = 0644 - - TmpDir = "/tmp/kubekey/" - - // command - CopyCmd = "cp -r %s %s" - MoveCmd = "mv -f %s %s" -) diff --git a/cmd/kk/pkg/core/connector/dialer.go b/cmd/kk/pkg/core/connector/dialer.go deleted file mode 100644 index 5fc77e35..00000000 --- a/cmd/kk/pkg/core/connector/dialer.go +++ /dev/null @@ -1,86 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package connector - -import ( - "sync" - "time" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -type Dialer struct { - lock sync.Mutex - connections map[string]Connection -} - -func NewDialer() *Dialer { - return &Dialer{ - connections: make(map[string]Connection), - } -} - -func (d *Dialer) Connect(host Host) (Connection, error) { - var err error - - d.lock.Lock() - defer d.lock.Unlock() - - conn, ok := d.connections[host.GetName()] - if !ok { - opts := Cfg{ - Username: host.GetUser(), - Port: host.GetPort(), - Address: host.GetAddress(), - Password: host.GetPassword(), - PrivateKey: host.GetPrivateKey(), - KeyFile: host.GetPrivateKeyPath(), - Timeout: time.Duration(host.GetTimeout()) * time.Second, - } - conn, err = NewConnection(opts) - if err != nil { - return nil, err - } - d.connections[host.GetName()] = conn - } - - return conn, nil -} - -func (d *Dialer) Close(host Host) { - conn, ok := d.connections[host.GetName()] - if !ok { - return - } - - conn.Close() - logger.Log.Debugf("close connection %s", host.GetName()) - - c := conn.(*connection) - d.forgetConnection(c) -} - -func (d *Dialer) forgetConnection(conn *connection) { - d.lock.Lock() - defer d.lock.Unlock() - - for k := range d.connections { - if d.connections[k] == conn { - delete(d.connections, k) - } - } -} diff --git a/cmd/kk/pkg/core/connector/host.go b/cmd/kk/pkg/core/connector/host.go deleted file mode 100644 index 762c5a89..00000000 --- a/cmd/kk/pkg/core/connector/host.go +++ /dev/null @@ -1,154 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package connector - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" -) - -type BaseHost struct { - Name string `yaml:"name,omitempty" json:"name,omitempty"` - Address string `yaml:"address,omitempty" json:"address,omitempty"` - InternalAddress string `yaml:"internalAddress,omitempty" json:"internalAddress,omitempty"` - Port int `yaml:"port,omitempty" json:"port,omitempty"` - User string `yaml:"user,omitempty" json:"user,omitempty"` - Password string `yaml:"password,omitempty" json:"password,omitempty"` - PrivateKey string `yaml:"privateKey,omitempty" json:"privateKey,omitempty"` - PrivateKeyPath string `yaml:"privateKeyPath,omitempty" json:"privateKeyPath,omitempty"` - Arch string `yaml:"arch,omitempty" json:"arch,omitempty"` - Timeout int64 `yaml:"timeout,omitempty" json:"timeout,omitempty"` - - Roles []string `json:"-"` - RoleTable map[string]bool `json:"-"` - Cache *cache.Cache `json:"-"` -} - -func NewHost() *BaseHost { - return &BaseHost{ - Roles: make([]string, 0, 0), - RoleTable: make(map[string]bool), - Cache: cache.NewCache(), - } -} - -func (b *BaseHost) GetName() string { - return b.Name -} - -func (b *BaseHost) SetName(name string) { - b.Name = name -} - -func (b *BaseHost) GetAddress() string { - return b.Address -} - -func (b *BaseHost) SetAddress(str string) { - b.Address = str -} - -func (b *BaseHost) GetInternalAddress() string { - return b.InternalAddress -} - -func (b *BaseHost) SetInternalAddress(str string) { - b.InternalAddress = str -} - -func (b *BaseHost) GetPort() int { - return b.Port -} - -func (b *BaseHost) SetPort(port int) { - b.Port = port -} - -func (b *BaseHost) GetUser() string { - return b.User -} - -func (b *BaseHost) SetUser(u string) { - b.User = u -} - -func (b *BaseHost) GetPassword() string { - return b.Password -} - -func (b *BaseHost) SetPassword(password string) { - b.Password = password -} - -func (b *BaseHost) GetPrivateKey() string { - return b.PrivateKey -} - -func (b *BaseHost) SetPrivateKey(privateKey string) { - b.PrivateKey = privateKey -} - -func (b *BaseHost) GetPrivateKeyPath() string { - return b.PrivateKeyPath -} - -func (b *BaseHost) SetPrivateKeyPath(path string) { - b.PrivateKeyPath = path -} - -func (b *BaseHost) GetArch() string { - return b.Arch -} - -func (b *BaseHost) SetArch(arch string) { - b.Arch = arch -} - -func (b *BaseHost) GetTimeout() int64 { - return b.Timeout -} - -func (b *BaseHost) SetTimeout(timeout int64) { - b.Timeout = timeout -} - -func (b *BaseHost) GetRoles() []string { - return b.Roles -} - -func (b *BaseHost) SetRoles(roles []string) { - b.Roles = roles -} - -func (b *BaseHost) SetRole(role string) { - b.RoleTable[role] = true - b.Roles = append(b.Roles, role) -} - -func (b *BaseHost) IsRole(role string) bool { - if res, ok := b.RoleTable[role]; ok { - return res - } - return false -} - -func (b *BaseHost) GetCache() *cache.Cache { - return b.Cache -} - -func (b *BaseHost) SetCache(c *cache.Cache) { - b.Cache = c -} diff --git a/cmd/kk/pkg/core/connector/interface.go b/cmd/kk/pkg/core/connector/interface.go deleted file mode 100644 index 60d12418..00000000 --- a/cmd/kk/pkg/core/connector/interface.go +++ /dev/null @@ -1,93 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package connector - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "io" - "os" -) - -type Connection interface { - Exec(cmd string, host Host) (stdout string, code int, err error) - PExec(cmd string, stdin io.Reader, stdout io.Writer, stderr io.Writer, host Host) (code int, err error) - Fetch(local, remote string, host Host) error - Scp(local, remote string, host Host) error - RemoteFileExist(remote string, host Host) bool - RemoteDirExist(remote string, host Host) (bool, error) - MkDirAll(path string, mode string, host Host) error - Chmod(path string, mode os.FileMode) error - Close() -} - -type Connector interface { - Connect(host Host) (Connection, error) - Close(host Host) -} - -type ModuleRuntime interface { - GetObjName() string - SetObjName(name string) - GenerateWorkDir() error - GetHostWorkDir() string - GetWorkDir() string - GetIgnoreErr() bool - GetAllHosts() []Host - SetAllHosts([]Host) - GetHostsByRole(role string) []Host - DeleteHost(host Host) - HostIsDeprecated(host Host) bool - InitLogger() error -} - -type Runtime interface { - GetRunner() *Runner - SetRunner(r *Runner) - GetConnector() Connector - SetConnector(c Connector) - RemoteHost() Host - Copy() Runtime - ModuleRuntime -} - -type Host interface { - GetName() string - SetName(name string) - GetAddress() string - SetAddress(str string) - GetInternalAddress() string - SetInternalAddress(str string) - GetPort() int - SetPort(port int) - GetUser() string - SetUser(u string) - GetPassword() string - SetPassword(password string) - GetPrivateKey() string - SetPrivateKey(privateKey string) - GetPrivateKeyPath() string - SetPrivateKeyPath(path string) - GetArch() string - SetArch(arch string) - GetTimeout() int64 - SetTimeout(timeout int64) - GetRoles() []string - SetRoles(roles []string) - IsRole(role string) bool - GetCache() *cache.Cache - SetCache(c *cache.Cache) -} diff --git a/cmd/kk/pkg/core/connector/runner.go b/cmd/kk/pkg/core/connector/runner.go deleted file mode 100644 index eea7ccc2..00000000 --- a/cmd/kk/pkg/core/connector/runner.go +++ /dev/null @@ -1,191 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package connector - -import ( - "errors" - "fmt" - "os" - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type Runner struct { - Conn Connection - Debug bool - Host Host - Index int -} - -func (r *Runner) Exec(cmd string, printOutput bool) (string, int, error) { - if r.Conn == nil { - return "", 1, errors.New("no ssh connection available") - } - - stdout, code, err := r.Conn.Exec(cmd, r.Host) - logger.Log.Debugf("command: [%s]\n%s", r.Host.GetName(), cmd) - if stdout != "" { - logger.Log.Debugf("stdout: [%s]\n%s", r.Host.GetName(), stdout) - } - if err != nil { - logger.Log.Debugf("stderr: [%s]\n%s", r.Host.GetName(), err) - } - - if printOutput { - if stdout != "" { - logger.Log.Infof("stdout: [%s]\n%s", r.Host.GetName(), stdout) - } - } - return stdout, code, err -} - -func (r *Runner) Cmd(cmd string, printOutput bool) (string, error) { - stdout, _, err := r.Exec(cmd, printOutput) - if err != nil { - return stdout, err - } - return stdout, nil -} - -func (r *Runner) SudoExec(cmd string, printOutput bool) (string, int, error) { - return r.Exec(SudoPrefix(cmd), printOutput) -} - -func (r *Runner) SudoCmd(cmd string, printOutput bool) (string, error) { - return r.Cmd(SudoPrefix(cmd), printOutput) -} - -func (r *Runner) Fetch(local, remote string) error { - if r.Conn == nil { - return errors.New("no ssh connection available") - } - - if err := r.Conn.Fetch(local, remote, r.Host); err != nil { - logger.Log.Debugf("fetch remote file %s to local %s failed: %v", remote, local, err) - return err - } - logger.Log.Debugf("fetch remote file %s to local %s success", remote, local) - return nil -} - -func (r *Runner) Scp(local, remote string) error { - if r.Conn == nil { - return errors.New("no ssh connection available") - } - - if err := r.Conn.Scp(local, remote, r.Host); err != nil { - logger.Log.Debugf("scp local file %s to remote %s failed: %v", local, remote, err) - return err - } - logger.Log.Debugf("scp local file %s to remote %s success", local, remote) - return nil -} - -func (r *Runner) SudoScp(local, remote string) error { - if r.Conn == nil { - return errors.New("no ssh connection available") - } - - // scp to tmp dir - remoteTmp := filepath.Join(common.TmpDir, remote) - //remoteTmp := remote - if err := r.Scp(local, remoteTmp); err != nil { - return err - } - - baseRemotePath := remote - if !util.IsDir(local) { - baseRemotePath = filepath.Dir(remote) - } - if err := r.Conn.MkDirAll(baseRemotePath, "", r.Host); err != nil { - return err - } - - if _, err := r.SudoCmd(fmt.Sprintf(common.MoveCmd, remoteTmp, remote), false); err != nil { - return err - } - - if _, err := r.SudoCmd(fmt.Sprintf("rm -rf %s", filepath.Join(common.TmpDir, "*")), false); err != nil { - return err - } - return nil -} - -func (r *Runner) FileExist(remote string) (bool, error) { - if r.Conn == nil { - return false, errors.New("no ssh connection available") - } - - ok := r.Conn.RemoteFileExist(remote, r.Host) - logger.Log.Debugf("check remote file exist: %v", ok) - return ok, nil -} - -func (r *Runner) DirExist(remote string) (bool, error) { - if r.Conn == nil { - return false, errors.New("no ssh connection available") - } - - ok, err := r.Conn.RemoteDirExist(remote, r.Host) - if err != nil { - logger.Log.Debugf("check remote dir exist failed: %v", err) - return false, err - } - logger.Log.Debugf("check remote dir exist: %v", ok) - return ok, nil -} - -func (r *Runner) MkDir(path string) error { - if r.Conn == nil { - return errors.New("no ssh connection available") - } - - if err := r.Conn.MkDirAll(path, "", r.Host); err != nil { - logger.Log.Errorf("make remote dir %s failed: %v", path, err) - return err - } - return nil -} - -func (r *Runner) Chmod(path string, mode os.FileMode) error { - if r.Conn == nil { - return errors.New("no ssh connection available") - } - - if err := r.Conn.Chmod(path, mode); err != nil { - logger.Log.Errorf("chmod remote path %s failed: %v", path, err) - return err - } - return nil -} - -func (r *Runner) FileMd5(path string) (string, error) { - if r.Conn == nil { - return "", errors.New("no ssh connection available") - } - - cmd := fmt.Sprintf("md5sum %s | cut -d\" \" -f1", path) - out, _, err := r.Conn.Exec(cmd, r.Host) - if err != nil { - logger.Log.Errorf("count remote %s md5 failed: %v", path, err) - return "", err - } - return out, nil -} diff --git a/cmd/kk/pkg/core/connector/runtime.go b/cmd/kk/pkg/core/connector/runtime.go deleted file mode 100644 index 48d32f0e..00000000 --- a/cmd/kk/pkg/core/connector/runtime.go +++ /dev/null @@ -1,225 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package connector - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type BaseRuntime struct { - ObjName string - connector Connector - runner *Runner - workDir string - verbose bool - ignoreErr bool - allHosts []Host - roleHosts map[string][]Host - deprecatedHosts map[string]string -} - -func NewBaseRuntime(name string, connector Connector, verbose bool, ignoreErr bool) BaseRuntime { - base := BaseRuntime{ - ObjName: name, - connector: connector, - verbose: verbose, - ignoreErr: ignoreErr, - allHosts: make([]Host, 0, 0), - roleHosts: make(map[string][]Host), - deprecatedHosts: make(map[string]string), - } - if err := base.GenerateWorkDir(); err != nil { - fmt.Printf("[ERRO]: Failed to create KubeKey work dir: %s\n", err) - os.Exit(1) - } - if err := base.InitLogger(); err != nil { - fmt.Printf("[ERRO]: Failed to init KubeKey log entry: %s\n", err) - os.Exit(1) - } - return base -} - -func (b *BaseRuntime) GetObjName() string { - return b.ObjName -} - -func (b *BaseRuntime) SetObjName(name string) { - b.ObjName = name -} - -func (b *BaseRuntime) GetRunner() *Runner { - return b.runner -} - -func (b *BaseRuntime) SetRunner(r *Runner) { - b.runner = r -} - -func (b *BaseRuntime) GetConnector() Connector { - return b.connector -} - -func (b *BaseRuntime) SetConnector(c Connector) { - b.connector = c -} - -func (b *BaseRuntime) GenerateWorkDir() error { - currentDir, err := filepath.Abs(filepath.Dir(os.Args[0])) - if err != nil { - return errors.Wrap(err, "get current dir failed") - } - - rootPath := filepath.Join(currentDir, common.KubeKey) - if err := util.CreateDir(rootPath); err != nil { - return errors.Wrap(err, "create work dir failed") - } - b.workDir = rootPath - - logDir := filepath.Join(rootPath, "logs") - if err := util.CreateDir(logDir); err != nil { - return errors.Wrap(err, "create logs dir failed") - } - - for i := range b.allHosts { - subPath := filepath.Join(rootPath, b.allHosts[i].GetName()) - if err := util.CreateDir(subPath); err != nil { - return errors.Wrap(err, "create work dir failed") - } - } - return nil -} - -func (b *BaseRuntime) GetHostWorkDir() string { - return filepath.Join(b.workDir, b.RemoteHost().GetName()) -} - -func (b *BaseRuntime) GetWorkDir() string { - return b.workDir -} - -func (b *BaseRuntime) GetIgnoreErr() bool { - return b.ignoreErr -} - -func (b *BaseRuntime) GetAllHosts() []Host { - hosts := make([]Host, 0, 0) - for i := range b.allHosts { - if b.allHosts[i] == nil || b.HostIsDeprecated(b.allHosts[i]) { - continue - } - hosts = append(hosts, b.allHosts[i]) - } - return hosts -} - -func (b *BaseRuntime) SetAllHosts(hosts []Host) { - b.allHosts = hosts -} - -func (b *BaseRuntime) GetHostsByRole(role string) []Host { - if _, ok := b.roleHosts[role]; ok { - return b.roleHosts[role] - } else { - return []Host{} - } -} - -func (b *BaseRuntime) RemoteHost() Host { - return b.GetRunner().Host -} - -func (b *BaseRuntime) DeleteHost(host Host) { - i := 0 - for j := range b.allHosts { - if b.allHosts[j].GetName() != host.GetName() { - b.allHosts[i] = b.allHosts[j] - i++ - } - } - b.allHosts[i] = nil - b.allHosts = b.allHosts[:i] - b.RoleMapDelete(host) - b.deprecatedHosts[host.GetName()] = "" -} - -func (b *BaseRuntime) HostIsDeprecated(host Host) bool { - if _, ok := b.deprecatedHosts[host.GetName()]; ok { - return true - } - return false -} - -func (b *BaseRuntime) InitLogger() error { - if b.GetWorkDir() == "" { - if err := b.GenerateWorkDir(); err != nil { - return err - } - } - logDir := filepath.Join(b.GetWorkDir(), "logs") - logger.Log = logger.NewLogger(logDir, b.verbose) - return nil -} - -func (b *BaseRuntime) Copy() Runtime { - runtime := *b - return &runtime -} - -func (b *BaseRuntime) GenerateRoleMap() { - for i := range b.allHosts { - b.AppendRoleMap(b.allHosts[i]) - } -} - -func (b *BaseRuntime) AppendHost(host Host) { - b.allHosts = append(b.allHosts, host) -} - -func (b *BaseRuntime) AppendRoleMap(host Host) { - for _, r := range host.GetRoles() { - if hosts, ok := b.roleHosts[r]; ok { - hosts = append(hosts, host) - b.roleHosts[r] = hosts - } else { - first := make([]Host, 0, 0) - first = append(first, host) - b.roleHosts[r] = first - } - } -} - -func (b *BaseRuntime) RoleMapDelete(host Host) { - for role, hosts := range b.roleHosts { - i := 0 - for j := range hosts { - if hosts[j].GetName() != host.GetName() { - hosts[i] = hosts[j] - i++ - } - } - hosts = hosts[:i] - b.roleHosts[role] = hosts - } -} diff --git a/cmd/kk/pkg/core/connector/ssh.go b/cmd/kk/pkg/core/connector/ssh.go deleted file mode 100644 index 261ee7f0..00000000 --- a/cmd/kk/pkg/core/connector/ssh.go +++ /dev/null @@ -1,608 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package connector - -import ( - "bufio" - "context" - "encoding/base64" - "fmt" - "io" - "net" - "os" - "path" - "path/filepath" - "strconv" - "strings" - "sync" - "time" - - "github.com/pkg/errors" - "github.com/pkg/sftp" - "golang.org/x/crypto/ssh" - "golang.org/x/crypto/ssh/agent" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type Cfg struct { - Username string - Password string - Address string - Port int - PrivateKey string - KeyFile string - AgentSocket string - Timeout time.Duration - Bastion string - BastionPort int - BastionUser string -} - -const socketEnvPrefix = "env:" - -type connection struct { - mu sync.Mutex - sftpclient *sftp.Client - sshclient *ssh.Client - ctx context.Context - cancel context.CancelFunc -} - -func NewConnection(cfg Cfg) (Connection, error) { - cfg, err := validateOptions(cfg) - if err != nil { - return nil, errors.Wrap(err, "Failed to validate ssh connection parameters") - } - - authMethods := make([]ssh.AuthMethod, 0) - - if len(cfg.Password) > 0 { - authMethods = append(authMethods, ssh.Password(cfg.Password)) - } - - if len(cfg.PrivateKey) > 0 { - signer, parseErr := ssh.ParsePrivateKey([]byte(cfg.PrivateKey)) - if parseErr != nil { - return nil, errors.Wrap(parseErr, "The given SSH key could not be parsed") - } - authMethods = append(authMethods, ssh.PublicKeys(signer)) - } - - if len(cfg.AgentSocket) > 0 { - addr := cfg.AgentSocket - - if strings.HasPrefix(cfg.AgentSocket, socketEnvPrefix) { - envName := strings.TrimPrefix(cfg.AgentSocket, socketEnvPrefix) - - if envAddr := os.Getenv(envName); len(envAddr) > 0 { - addr = envAddr - } - } - - socket, dialErr := net.Dial("unix", addr) - if dialErr != nil { - return nil, errors.Wrapf(dialErr, "could not open socket %q", addr) - } - - agentClient := agent.NewClient(socket) - - signers, signersErr := agentClient.Signers() - if signersErr != nil { - _ = socket.Close() - return nil, errors.Wrap(signersErr, "error when creating signer for SSH agent") - } - - authMethods = append(authMethods, ssh.PublicKeys(signers...)) - } - - sshConfig := &ssh.ClientConfig{ - User: cfg.Username, - Timeout: cfg.Timeout, - Auth: authMethods, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - targetHost := cfg.Address - targetPort := strconv.Itoa(cfg.Port) - - if cfg.Bastion != "" { - targetHost = cfg.Bastion - targetPort = strconv.Itoa(cfg.BastionPort) - sshConfig.User = cfg.BastionUser - } - - endpoint := net.JoinHostPort(targetHost, targetPort) - - client, err := ssh.Dial("tcp", endpoint, sshConfig) - if err != nil { - return nil, errors.Wrapf(err, "could not establish connection to %s", endpoint) - } - - ctx, cancelFn := context.WithCancel(context.Background()) - sshConn := &connection{ - ctx: ctx, - cancel: cancelFn, - } - - if cfg.Bastion == "" { - sshConn.sshclient = client - sftpClient, err := sftp.NewClient(sshConn.sshclient) - if err != nil { - return nil, errors.Wrapf(err, "new sftp client failed: %v", err) - } - sshConn.sftpclient = sftpClient - return sshConn, nil - } - - endpointBehindBastion := net.JoinHostPort(cfg.Address, strconv.Itoa(cfg.Port)) - - conn, err := client.Dial("tcp", endpointBehindBastion) - if err != nil { - return nil, errors.Wrapf(err, "could not establish connection to %s", endpointBehindBastion) - } - - sshConfig.User = cfg.Username - ncc, chans, reqs, err := ssh.NewClientConn(conn, endpointBehindBastion, sshConfig) - if err != nil { - return nil, errors.Wrapf(err, "could not establish connection to %s", endpointBehindBastion) - } - - sshConn.sshclient = ssh.NewClient(ncc, chans, reqs) - sftpClient, err := sftp.NewClient(sshConn.sshclient) - if err != nil { - return nil, errors.Wrapf(err, "new sftp client failed: %v", err) - } - sshConn.sftpclient = sftpClient - return sshConn, nil -} - -func validateOptions(cfg Cfg) (Cfg, error) { - if len(cfg.Username) == 0 { - return cfg, errors.New("No username specified for SSH connection") - } - - if len(cfg.Address) == 0 { - return cfg, errors.New("No address specified for SSH connection") - } - - if len(cfg.Password) == 0 && len(cfg.PrivateKey) == 0 && len(cfg.KeyFile) == 0 && len(cfg.AgentSocket) == 0 { - return cfg, errors.New("Must specify at least one of password, private key, keyfile or agent socket") - } - - if len(cfg.PrivateKey) == 0 && len(cfg.KeyFile) > 0 { - content, err := os.ReadFile(cfg.KeyFile) - if err != nil { - return cfg, errors.Wrapf(err, "Failed to read keyfile %q", cfg.KeyFile) - } - - cfg.PrivateKey = string(content) - cfg.KeyFile = "" - } - - if cfg.Port <= 0 { - cfg.Port = 22 - } - - if cfg.BastionPort <= 0 { - cfg.BastionPort = 22 - } - - if cfg.BastionUser == "" { - cfg.BastionUser = cfg.Username - } - - if cfg.Timeout == 0 { - cfg.Timeout = 15 * time.Second - } - - return cfg, nil -} - -func (c *connection) Close() { - c.mu.Lock() - defer c.mu.Unlock() - - if c.sshclient == nil && c.sftpclient == nil { - return - } - c.cancel() - - if c.sshclient != nil { - c.sshclient.Close() - c.sshclient = nil - } - if c.sftpclient != nil { - c.sftpclient.Close() - c.sftpclient = nil - } -} - -func (c *connection) session() (*ssh.Session, error) { - c.mu.Lock() - defer c.mu.Unlock() - - if c.sshclient == nil { - return nil, errors.New("connection closed") - } - - sess, err := c.sshclient.NewSession() - if err != nil { - return nil, err - } - - modes := ssh.TerminalModes{ - ssh.ECHO: 0, // disable echoing - ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud - ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud - } - - err = sess.RequestPty("xterm", 100, 50, modes) - if err != nil { - return nil, err - } - - return sess, nil -} - -func (c *connection) PExec(cmd string, stdin io.Reader, stdout io.Writer, stderr io.Writer, host Host) (int, error) { - sess, err := c.session() - if err != nil { - return 1, errors.Wrap(err, "failed to get SSH session") - } - defer sess.Close() - - sess.Stdin = stdin - sess.Stdout = stdout - sess.Stderr = stderr - - exitCode := 0 - - in, _ := sess.StdinPipe() - out, _ := sess.StdoutPipe() - - err = sess.Start(strings.TrimSpace(cmd)) - if err != nil { - exitCode = -1 - if exitErr, ok := err.(*ssh.ExitError); ok { - exitCode = exitErr.ExitStatus() - } - } - - var ( - output []byte - line = "" - r = bufio.NewReader(out) - ) - - for { - b, err := r.ReadByte() - if err != nil { - break - } - - output = append(output, b) - - if b == byte('\n') { - line = "" - continue - } - - line += string(b) - - if (strings.HasPrefix(line, "[sudo] password for ") || strings.HasPrefix(line, "Password")) && strings.HasSuffix(line, ": ") { - _, err = in.Write([]byte(host.GetPassword() + "\n")) - if err != nil { - break - } - } - } - err = sess.Wait() - if err != nil { - exitCode = -1 - if exitErr, ok := err.(*ssh.ExitError); ok { - exitCode = exitErr.ExitStatus() - } - } - - // preserve original error - return exitCode, err -} - -func (c *connection) Exec(cmd string, host Host) (stdout string, code int, err error) { - sess, err := c.session() - if err != nil { - return "", 1, errors.Wrap(err, "failed to get SSH session") - } - defer sess.Close() - - exitCode := 0 - - in, _ := sess.StdinPipe() - out, _ := sess.StdoutPipe() - - err = sess.Start(strings.TrimSpace(cmd)) - if err != nil { - exitCode = -1 - if exitErr, ok := err.(*ssh.ExitError); ok { - exitCode = exitErr.ExitStatus() - } - return "", exitCode, err - } - - var ( - output []byte - line = "" - r = bufio.NewReader(out) - ) - - for { - b, err := r.ReadByte() - if err != nil { - break - } - - output = append(output, b) - - if b == byte('\n') { - line = "" - continue - } - - line += string(b) - - if (strings.HasPrefix(line, "[sudo] password for ") || strings.HasPrefix(line, "Password")) && strings.HasSuffix(line, ": ") { - _, err = in.Write([]byte(host.GetPassword() + "\n")) - if err != nil { - break - } - } - } - err = sess.Wait() - if err != nil { - exitCode = -1 - if exitErr, ok := err.(*ssh.ExitError); ok { - exitCode = exitErr.ExitStatus() - } - } - outStr := strings.TrimPrefix(string(output), fmt.Sprintf("[sudo] password for %s:", host.GetUser())) - - // preserve original error - return strings.TrimSpace(outStr), exitCode, errors.Wrapf(err, "Failed to exec command: %s \n%s", cmd, strings.TrimSpace(outStr)) -} - -func (c *connection) Fetch(local, remote string, host Host) error { - //srcFile, err := c.sftpclient.Open(remote) - //if err != nil { - // return fmt.Errorf("open remote file failed %v, remote path: %s", err, remote) - //} - //defer srcFile.Close() - - // Base64 encoding is performed on the contents of the file to prevent garbled code in the target file. - output, _, err := c.Exec(SudoPrefix(fmt.Sprintf("cat %s | base64 -w 0", remote)), host) - if err != nil { - return fmt.Errorf("open remote file failed %v, remote path: %s", err, remote) - } - - err = util.MkFileFullPathDir(local) - if err != nil { - return err - } - // open local Destination file - dstFile, err := os.Create(local) - if err != nil { - return fmt.Errorf("create local file failed %v", err) - } - defer dstFile.Close() - // copy to local file - //_, err = srcFile.WriteTo(dstFile) - if base64Str, err := base64.StdEncoding.DecodeString(output); err != nil { - return err - } else { - if _, err = dstFile.WriteString(string(base64Str)); err != nil { - return err - } - } - - return nil -} - -type scpErr struct { - err error -} - -func (c *connection) Scp(src, dst string, host Host) error { - baseRemotePath := filepath.Dir(dst) - - if err := c.MkDirAll(baseRemotePath, "777", host); err != nil { - return err - } - f, err := os.Stat(src) - if err != nil { - return fmt.Errorf("get file stat failed: %s", err) - } - - number := 1 - if f.IsDir() { - number = util.CountDirFiles(src) - } - // empty dir - if number == 0 { - return nil - } - - scpErr := new(scpErr) - if f.IsDir() { - c.copyDirToRemote(src, dst, scpErr, host) - if scpErr.err != nil { - return scpErr.err - } - } else { - if err := c.copyFileToRemote(src, dst, host); err != nil { - return err - } - } - return nil -} - -func (c *connection) copyDirToRemote(src, dst string, scrErr *scpErr, host Host) { - localFiles, err := os.ReadDir(src) - if err != nil { - logger.Log.Errorf("read local path dir %s failed %v", src, err) - scrErr.err = err - return - } - if err = c.MkDirAll(dst, "", host); err != nil { - logger.Log.Errorf("failed to create remote path %s:%v", dst, err) - scrErr.err = err - return - } - for _, file := range localFiles { - local := path.Join(src, file.Name()) - remote := path.Join(dst, file.Name()) - if file.IsDir() { - if err = c.MkDirAll(remote, "", host); err != nil { - logger.Log.Errorf("failed to create remote path %s:%v", remote, err) - scrErr.err = err - return - } - c.copyDirToRemote(local, remote, scrErr, host) - } else { - err := c.copyFileToRemote(local, remote, host) - if err != nil { - logger.Log.Errorf("copy local file %s to remote file %s failed %v ", local, remote, err) - scrErr.err = err - return - } - } - } -} - -func (c *connection) copyFileToRemote(src, dst string, host Host) error { - // check remote file md5 first - var ( - srcMd5, dstMd5 string - ) - srcMd5 = util.LocalMd5Sum(src) - if c.RemoteFileExist(dst, host) { - dstMd5 = c.RemoteMd5Sum(dst, host) - if srcMd5 == dstMd5 { - logger.Log.Debug("remote file %s md5 value is the same as local file, skip scp", dst) - return nil - } - } - - srcFile, err := os.Open(src) - if err != nil { - return err - } - defer srcFile.Close() - // the dst file mod will be 0666 - dstFile, err := c.sftpclient.Create(dst) - if err != nil { - return err - } - fileStat, err := srcFile.Stat() - if err != nil { - return fmt.Errorf("get file stat failed %v", err) - } - if err := dstFile.Chmod(fileStat.Mode()); err != nil { - return fmt.Errorf("chmod remote file failed %v", err) - } - defer dstFile.Close() - _, err = io.Copy(dstFile, srcFile) - if err != nil { - return err - } - dstMd5 = c.RemoteMd5Sum(dst, host) - if srcMd5 != dstMd5 { - return fmt.Errorf("validate md5sum failed %s != %s", srcMd5, dstMd5) - } - return nil -} - -func (c *connection) RemoteMd5Sum(dst string, host Host) string { - cmd := fmt.Sprintf("md5sum %s | cut -d\" \" -f1", dst) - remoteMd5, _, err := c.Exec(cmd, host) - if err != nil { - logger.Log.Errorf("exec countRemoteMd5Command %s failed: %v", cmd, err) - } - return remoteMd5 -} - -func (c *connection) RemoteFileExist(dst string, host Host) bool { - remoteFileName := path.Base(dst) - remoteFileDirName := path.Dir(dst) - - remoteFileCommand := fmt.Sprintf(SudoPrefix("ls -l %s/%s 2>/dev/null |wc -l"), remoteFileDirName, remoteFileName) - - out, _, err := c.Exec(remoteFileCommand, host) - defer func() { - if r := recover(); r != nil { - logger.Log.Errorf("exec remoteFileCommand %s err: %v", remoteFileCommand, err) - } - }() - if err != nil { - panic(1) - } - count, err := strconv.Atoi(strings.TrimSpace(out)) - defer func() { - if r := recover(); r != nil { - logger.Log.Errorf("check remote file exist err: %v", err) - } - }() - if err != nil { - panic(1) - } - return count != 0 -} - -func (c *connection) RemoteDirExist(dst string, host Host) (bool, error) { - if _, err := c.sftpclient.ReadDir(dst); err != nil { - return false, err - } - return true, nil -} - -func (c *connection) MkDirAll(path string, mode string, host Host) error { - if mode == "" { - mode = "775" - } - mkDstDir := fmt.Sprintf("mkdir -p -m %s %s || true", mode, path) - if strings.Contains(path, common.TmpDir) { - mkDstDir = fmt.Sprintf("mkdir -p %s && chmod -R %s %s || true", path, mode, common.TmpDir) - } - if _, _, err := c.Exec(SudoPrefix(mkDstDir), host); err != nil { - return err - } - - return nil -} - -func (c *connection) Chmod(path string, mode os.FileMode) error { - remotePath := filepath.Dir(path) - if err := c.sftpclient.Chmod(remotePath, mode); err != nil { - return err - } - return nil -} - -func SudoPrefix(cmd string) string { - return fmt.Sprintf("sudo -E /bin/bash -c \"%s\"", cmd) -} diff --git a/cmd/kk/pkg/core/connector/tee.go b/cmd/kk/pkg/core/connector/tee.go deleted file mode 100644 index 2ba468e7..00000000 --- a/cmd/kk/pkg/core/connector/tee.go +++ /dev/null @@ -1,48 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package connector - -import ( - "bytes" - "io" - "strings" -) - -type Tee struct { - buffer bytes.Buffer - upstream io.WriteCloser -} - -// NewTee constructor -func NewTee(wc io.WriteCloser) *Tee { - return &Tee{upstream: wc} -} - -func (t *Tee) Write(p []byte) (int, error) { - t.buffer.Write(p) - - return t.upstream.Write(p) -} - -func (t *Tee) String() string { - return strings.TrimSpace(t.buffer.String()) -} - -// Close underlying io.Closer -func (t *Tee) Close() error { - return t.upstream.Close() -} diff --git a/cmd/kk/pkg/core/ending/action_result.go b/cmd/kk/pkg/core/ending/action_result.go deleted file mode 100644 index 7f26d92a..00000000 --- a/cmd/kk/pkg/core/ending/action_result.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package ending - -import ( - "time" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type ActionResult struct { - Host connector.Host - Status ResultStatus - Error error - StartTime time.Time - EndTime time.Time -} - -func (a *ActionResult) GetHost() connector.Host { - return a.Host -} - -func (a *ActionResult) GetStatus() ResultStatus { - return a.Status -} - -func (a *ActionResult) GetErr() error { - return a.Error -} - -func (a *ActionResult) GetStartTime() time.Time { - return a.StartTime -} - -func (a *ActionResult) GetEndTime() time.Time { - return a.EndTime -} diff --git a/cmd/kk/pkg/core/ending/interface.go b/cmd/kk/pkg/core/ending/interface.go deleted file mode 100644 index ee55753b..00000000 --- a/cmd/kk/pkg/core/ending/interface.go +++ /dev/null @@ -1,31 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package ending - -import ( - "time" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type Interface interface { - GetHost() connector.Host - GetStatus() ResultStatus - GetErr() error - GetStartTime() time.Time - GetEndTime() time.Time -} diff --git a/cmd/kk/pkg/core/ending/module_result.go b/cmd/kk/pkg/core/ending/module_result.go deleted file mode 100644 index 88d2757c..00000000 --- a/cmd/kk/pkg/core/ending/module_result.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package ending - -import ( - "time" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type ModuleResult struct { - HostResults map[string]Interface - CombineResult error - Status ResultStatus - StartTime time.Time - EndTime time.Time -} - -func NewModuleResult() *ModuleResult { - return &ModuleResult{HostResults: make(map[string]Interface), StartTime: time.Now(), Status: NULL} -} - -func (m *ModuleResult) IsFailed() bool { - return m.Status == FAILED -} - -func (m *ModuleResult) AppendHostResult(p Interface) { - if m.HostResults == nil { - return - } - m.HostResults[p.GetHost().GetName()] = p -} - -func (m *ModuleResult) LocalErrResult(err error) { - now := time.Now() - r := &ActionResult{ - Host: &connector.BaseHost{Name: common.LocalHost}, - Status: FAILED, - Error: err, - StartTime: m.StartTime, - EndTime: now, - } - - m.HostResults[r.Host.GetName()] = r - m.CombineResult = err - m.EndTime = now - m.Status = FAILED -} - -func (m *ModuleResult) ErrResult(combineErr error) { - m.EndTime = time.Now() - m.Status = FAILED - m.CombineResult = combineErr -} - -func (m *ModuleResult) NormalResult() { - m.EndTime = time.Now() - m.Status = SUCCESS -} diff --git a/cmd/kk/pkg/core/ending/status_enum.go b/cmd/kk/pkg/core/ending/status_enum.go deleted file mode 100644 index aa8779d1..00000000 --- a/cmd/kk/pkg/core/ending/status_enum.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package ending - -type ResultStatus int - -const ( - NULL ResultStatus = -99 - SKIPPED ResultStatus = iota - 2 - SUCCESS - FAILED -) - -var EnumList = []ResultStatus{ - NULL, - SKIPPED, - SUCCESS, - FAILED, -} - -func (r ResultStatus) String() string { - switch r { - case SUCCESS: - return "success" - case FAILED: - return "failed" - case SKIPPED: - return "skipped" - case NULL: - return "null" - default: - return "invalid option" - } -} - -func GetByCode(code int) ResultStatus { - switch code { - case -99: - return NULL - case -1: - return SKIPPED - case 0: - return SUCCESS - default: - return FAILED - } -} diff --git a/cmd/kk/pkg/core/ending/task_result.go b/cmd/kk/pkg/core/ending/task_result.go deleted file mode 100644 index 73565e95..00000000 --- a/cmd/kk/pkg/core/ending/task_result.go +++ /dev/null @@ -1,134 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package ending - -import ( - "fmt" - "sync" - "time" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type TaskResult struct { - mu sync.Mutex - ActionResults []*ActionResult - Status ResultStatus - StartTime time.Time - EndTime time.Time -} - -func NewTaskResult() *TaskResult { - return &TaskResult{ActionResults: make([]*ActionResult, 0, 0), Status: NULL, StartTime: time.Now()} -} - -func (t *TaskResult) ErrResult() { - if t.Status != NULL { - return - } - t.EndTime = time.Now() - t.Status = FAILED -} - -func (t *TaskResult) NormalResult() { - if t.Status != NULL { - return - } - t.EndTime = time.Now() - t.Status = SUCCESS -} - -func (t *TaskResult) SkippedResult() { - if t.Status != NULL { - return - } - t.EndTime = time.Now() - t.Status = SKIPPED -} - -func (t *TaskResult) AppendSkip(host connector.Host) { - t.mu.Lock() - defer t.mu.Unlock() - now := time.Now() - e := &ActionResult{ - Host: host, - Status: SKIPPED, - Error: nil, - StartTime: t.StartTime, - EndTime: now, - } - - t.ActionResults = append(t.ActionResults, e) - t.EndTime = now -} - -func (t *TaskResult) AppendSuccess(host connector.Host) { - t.mu.Lock() - defer t.mu.Unlock() - now := time.Now() - e := &ActionResult{ - Host: host, - Status: SUCCESS, - Error: nil, - StartTime: t.StartTime, - EndTime: now, - } - - t.ActionResults = append(t.ActionResults, e) - t.EndTime = now -} - -func (t *TaskResult) AppendErr(host connector.Host, err error) { - t.mu.Lock() - defer t.mu.Unlock() - now := time.Now() - e := &ActionResult{ - Host: host, - Status: FAILED, - Error: err, - StartTime: t.StartTime, - EndTime: now, - } - - t.ActionResults = append(t.ActionResults, e) - t.EndTime = now - t.Status = FAILED -} - -func (t *TaskResult) IsFailed() bool { - t.mu.Lock() - defer t.mu.Unlock() - return t.Status == FAILED -} - -func (t *TaskResult) CombineErr() error { - t.mu.Lock() - defer t.mu.Unlock() - if len(t.ActionResults) != 0 { - var str string - for i := range t.ActionResults { - if t.ActionResults[i].Status != FAILED { - continue - } - str += fmt.Sprintf("\nfailed: [%s] %s", t.ActionResults[i].Host.GetName(), t.ActionResults[i].Error.Error()) - } - return errors.New(str) - } - return nil -} diff --git a/cmd/kk/pkg/core/hook/handler.go b/cmd/kk/pkg/core/hook/handler.go deleted file mode 100644 index f5c61b13..00000000 --- a/cmd/kk/pkg/core/hook/handler.go +++ /dev/null @@ -1,23 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package hook - -func Call(hook Interface) error { - err := hook.Catch(hook.Try()) - hook.Finally() - return err -} diff --git a/cmd/kk/pkg/core/hook/interface.go b/cmd/kk/pkg/core/hook/interface.go deleted file mode 100644 index 4715f4bc..00000000 --- a/cmd/kk/pkg/core/hook/interface.go +++ /dev/null @@ -1,23 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package hook - -type Interface interface { - Try() error - Catch(err error) error - Finally() -} diff --git a/cmd/kk/pkg/core/logger/formatter.go b/cmd/kk/pkg/core/logger/formatter.go deleted file mode 100644 index 748d2785..00000000 --- a/cmd/kk/pkg/core/logger/formatter.go +++ /dev/null @@ -1,204 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package logger - -import ( - "bytes" - "fmt" - "github.com/sirupsen/logrus" - "runtime" - "sort" - "strings" - "time" -) - -type Formatter struct { - // TimestampFormat - default: time.StampMilli = "Jan _2 15:04:05.000" - TimestampFormat string - // NoColors - disable colors - NoColors bool - // ShowLevel - when the level < this field, it won't be show. default: TRACE - ShowLevel logrus.Level - // ShowFullLevel - show a full level [WARNING] instead of [WARN] - ShowFullLevel bool - // NoUppercaseLevel - no upper case for level value - NoUppercaseLevel bool - // HideKeys - show [fieldValue] instead of [fieldKey:fieldValue] - HideKeys bool - // FieldsDisplayWithOrder - default: all fields display and sorted alphabetically - FieldsDisplayWithOrder []string - // CallerFirst - print caller info first - CallerFirst bool - // CustomCallerFormatter - set custom formatter for caller info - CustomCallerFormatter func(*runtime.Frame) string -} - -func (f *Formatter) Format(entry *logrus.Entry) ([]byte, error) { - levelColor := getColorByLevel(entry.Level) - - timestampFormat := f.TimestampFormat - if timestampFormat == "" { - timestampFormat = time.StampMilli - } - - // output buffer - b := &bytes.Buffer{} - - // write time - b.WriteString(entry.Time.Format(timestampFormat)) - - if f.CallerFirst { - f.writeCaller(b, entry) - } - - if !f.NoColors { - fmt.Fprintf(b, "\x1b[%dm", levelColor) - } - - level := entry.Level - if f.ShowLevel >= level { - var levelStr string - if f.NoUppercaseLevel { - levelStr = entry.Level.String() - } else { - levelStr = strings.ToUpper(entry.Level.String()) - } - - b.WriteString(" [") - if f.ShowFullLevel { - b.WriteString(levelStr) - } else { - b.WriteString(levelStr[:4]) - } - b.WriteString("]") - } - - b.WriteString(" ") - - // write fields - if f.FieldsDisplayWithOrder == nil { - f.writeFields(b, entry) - } else { - f.writeOrderedFields(b, entry) - } - - b.WriteString(entry.Message) - - if !f.CallerFirst { - f.writeCaller(b, entry) - } - - b.WriteByte('\n') - - return b.Bytes(), nil -} - -const ( - colorRed = 31 - colorYellow = 33 - colorBlue = 36 - colorGray = 37 -) - -func getColorByLevel(level logrus.Level) int { - switch level { - case logrus.DebugLevel, logrus.TraceLevel: - return colorGray - case logrus.WarnLevel: - return colorYellow - case logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel: - return colorRed - default: - return colorBlue - } -} - -func (f *Formatter) writeFields(b *bytes.Buffer, entry *logrus.Entry) { - if len(entry.Data) != 0 { - fields := make([]string, 0, len(entry.Data)) - for field := range entry.Data { - fields = append(fields, field) - } - - sort.Strings(fields) - - b.WriteString("[") - for i, field := range fields { - f.writeField(b, entry, field, i) - } - b.WriteString("]") - } -} - -func (f *Formatter) writeOrderedFields(b *bytes.Buffer, entry *logrus.Entry) { - if len(entry.Data) != 0 { - b.WriteString("[") - length := len(entry.Data) - foundFieldsMap := map[string]bool{} - for i, field := range f.FieldsDisplayWithOrder { - if _, ok := entry.Data[field]; ok { - foundFieldsMap[field] = true - length-- - f.writeField(b, entry, field, i) - } - } - - if length > 0 { - notFoundFields := make([]string, 0, length) - for field := range entry.Data { - if foundFieldsMap[field] == false { - notFoundFields = append(notFoundFields, field) - } - } - - sort.Strings(notFoundFields) - - for i, field := range notFoundFields { - f.writeField(b, entry, field, i) - } - } - b.WriteString("]") - } -} - -func (f *Formatter) writeField(b *bytes.Buffer, entry *logrus.Entry, field string, i int) { - if f.HideKeys { - fmt.Fprintf(b, "%v", entry.Data[field]) - } else { - fmt.Fprintf(b, "%s:%v", field, entry.Data[field]) - } - - if i != len(entry.Data) && len(entry.Data) != 1 { - b.WriteString(" | ") - } -} - -func (f *Formatter) writeCaller(b *bytes.Buffer, entry *logrus.Entry) { - if entry.HasCaller() { - if f.CustomCallerFormatter != nil { - fmt.Fprintf(b, f.CustomCallerFormatter(entry.Caller)) - } else { - fmt.Fprintf( - b, - " (%s:%d %s)", - entry.Caller.File, - entry.Caller.Line, - entry.Caller.Function, - ) - } - } -} diff --git a/cmd/kk/pkg/core/logger/logger.go b/cmd/kk/pkg/core/logger/logger.go deleted file mode 100644 index ad3b2295..00000000 --- a/cmd/kk/pkg/core/logger/logger.go +++ /dev/null @@ -1,83 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package logger - -import ( - "fmt" - "path/filepath" - "time" - - rotatelogs "github.com/lestrrat-go/file-rotatelogs" - "github.com/rifflock/lfshook" - "github.com/sirupsen/logrus" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/common" -) - -var Log *KubeKeyLog - -type KubeKeyLog struct { - logrus.FieldLogger - OutputPath string - Verbose bool -} - -func NewLogger(outputPath string, verbose bool) *KubeKeyLog { - logger := logrus.New() - - formatter := &Formatter{ - HideKeys: true, - TimestampFormat: "15:04:05 MST", - NoColors: true, - ShowLevel: logrus.WarnLevel, - FieldsDisplayWithOrder: []string{common.Pipeline, common.Module, common.Task, common.Node}, - } - logger.SetFormatter(formatter) - - path := filepath.Join(outputPath, "./kubekey.log") - writer, _ := rotatelogs.New( - path+".%Y%m%d", - rotatelogs.WithLinkName(path), - rotatelogs.WithRotationTime(24*time.Hour), - ) - - logWriters := lfshook.WriterMap{ - logrus.InfoLevel: writer, - logrus.WarnLevel: writer, - logrus.ErrorLevel: writer, - logrus.FatalLevel: writer, - logrus.PanicLevel: writer, - } - - if verbose { - logger.SetLevel(logrus.DebugLevel) - logWriters[logrus.DebugLevel] = writer - } else { - logger.SetLevel(logrus.InfoLevel) - } - - logger.Hooks.Add(lfshook.NewHook(logWriters, formatter)) - return &KubeKeyLog{logger, outputPath, verbose} -} - -func (k *KubeKeyLog) Message(node, str string) { - Log.Infof("message: [%s]\n%s", node, str) -} - -func (k *KubeKeyLog) Messagef(node, format string, args ...interface{}) { - Log.Infof("message: [%s]\n%s", node, fmt.Sprintf(format, args...)) -} diff --git a/cmd/kk/pkg/core/logger/logger_test.go b/cmd/kk/pkg/core/logger/logger_test.go deleted file mode 100644 index 1370a135..00000000 --- a/cmd/kk/pkg/core/logger/logger_test.go +++ /dev/null @@ -1,42 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package logger - -import ( - "sync" - "testing" -) - -var log = NewLogger("", true) - -func TestKubeKey_Print(t *testing.T) { - wg := &sync.WaitGroup{} - for i := 0; i < 5; i++ { - - log.Info("empty fields") - l1 := *log - for j := 0; j < 10; j++ { - wg.Add(1) - go func(x int, log1 KubeKeyLog) { - log.Info("Congratulations!", "ssssss") - wg.Done() - }(j, l1) - } - wg.Wait() - } - -} diff --git a/cmd/kk/pkg/core/module/base.go b/cmd/kk/pkg/core/module/base.go deleted file mode 100644 index 7bfc84a8..00000000 --- a/cmd/kk/pkg/core/module/base.go +++ /dev/null @@ -1,85 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package module - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/hook" -) - -type BaseModule struct { - Name string - Desc string - Skip bool - ModuleCache *cache.Cache - PipelineCache *cache.Cache - Runtime connector.ModuleRuntime - PostHook []PostHookInterface -} - -func (b *BaseModule) IsSkip() bool { - return b.Skip -} - -func (b *BaseModule) Default(runtime connector.Runtime, pipelineCache *cache.Cache, moduleCache *cache.Cache) { - b.Runtime = runtime - b.PipelineCache = pipelineCache - b.ModuleCache = moduleCache -} - -func (b *BaseModule) Init() { - if b.Name == "" { - b.Name = DefaultModuleName - } -} - -func (b *BaseModule) Is() string { - return BaseModuleType -} - -func (b *BaseModule) Run(result *ending.ModuleResult) { - panic("implement me") -} - -func (b *BaseModule) Until() (*bool, error) { - return nil, nil -} - -func (b *BaseModule) Slogan() { -} - -func (b *BaseModule) AutoAssert() { -} - -func (b *BaseModule) AppendPostHook(h PostHookInterface) { - b.PostHook = append(b.PostHook, h) -} - -func (b *BaseModule) CallPostHook(result *ending.ModuleResult) error { - for i := range b.PostHook { - h := b.PostHook[i] - h.Init(b, result) - if err := hook.Call(h); err != nil { - return errors.Wrapf(err, "Module[%s] call post hook failed", b.Name) - } - } - return nil -} diff --git a/cmd/kk/pkg/core/module/const.go b/cmd/kk/pkg/core/module/const.go deleted file mode 100644 index 65e7b1f2..00000000 --- a/cmd/kk/pkg/core/module/const.go +++ /dev/null @@ -1,29 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package module - -const ( - DefaultModuleName = "DefaultBaseModule" - DefaultTaskModuleName = "DefaultTaskModule" - DefaultGoroutineModuleName = "DefaultGoroutineModuleName" - DefaultCustomModuleName = "DefaultCustomModule" - - BaseModuleType = "BaseModuleType" - TaskModuleType = "TaskModuleType" - GoroutineModuleType = "GoroutineModuleType" - CustomModuleType = "CustomModuleType" -) diff --git a/cmd/kk/pkg/core/module/custom_module.go b/cmd/kk/pkg/core/module/custom_module.go deleted file mode 100644 index 586644ab..00000000 --- a/cmd/kk/pkg/core/module/custom_module.go +++ /dev/null @@ -1,31 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package module - -type CustomModule struct { - BaseModule -} - -func (c *CustomModule) Init() { - if c.Name == "" { - c.Name = DefaultCustomModuleName - } -} - -func (c *CustomModule) Is() string { - return CustomModuleType -} diff --git a/cmd/kk/pkg/core/module/hook.go b/cmd/kk/pkg/core/module/hook.go deleted file mode 100644 index 58a45040..00000000 --- a/cmd/kk/pkg/core/module/hook.go +++ /dev/null @@ -1,48 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package module - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/hook" -) - -type PostHookInterface interface { - hook.Interface - Init(module Module, result *ending.ModuleResult) -} - -type PostHook struct { - Module Module - Result *ending.ModuleResult -} - -func (p *PostHook) Try() error { - panic("implement me") -} - -func (p *PostHook) Catch(err error) error { - return err -} - -func (p *PostHook) Finally() { -} - -func (p *PostHook) Init(module Module, result *ending.ModuleResult) { - p.Module = module - p.Result = result -} diff --git a/cmd/kk/pkg/core/module/interface.go b/cmd/kk/pkg/core/module/interface.go deleted file mode 100644 index 36a07334..00000000 --- a/cmd/kk/pkg/core/module/interface.go +++ /dev/null @@ -1,36 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package module - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" -) - -type Module interface { - IsSkip() bool - Default(runtime connector.Runtime, pipelineCache *cache.Cache, moduleCache *cache.Cache) - Init() - Is() string - Run(result *ending.ModuleResult) - Until() (*bool, error) - Slogan() - AutoAssert() - AppendPostHook(h PostHookInterface) - CallPostHook(result *ending.ModuleResult) error -} diff --git a/cmd/kk/pkg/core/module/task_module.go b/cmd/kk/pkg/core/module/task_module.go deleted file mode 100644 index 569302e0..00000000 --- a/cmd/kk/pkg/core/module/task_module.go +++ /dev/null @@ -1,77 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package module - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type BaseTaskModule struct { - BaseModule - Tasks []task.Interface -} - -func (b *BaseTaskModule) Init() { - if b.Name == "" { - b.Name = DefaultTaskModuleName - } -} - -func (b *BaseTaskModule) Is() string { - return TaskModuleType -} - -func (b *BaseTaskModule) Run(result *ending.ModuleResult) { - for i := range b.Tasks { - t := b.Tasks[i] - t.Init(b.Runtime.(connector.Runtime), b.ModuleCache, b.PipelineCache) - - logger.Log.Infof("[%s] %s", b.Name, t.GetDesc()) - res := t.Execute() - for j := range res.ActionResults { - ac := res.ActionResults[j] - logger.Log.Infof("%s: [%s]", ac.Status.String(), ac.Host.GetName()) - result.AppendHostResult(ac) - - if _, ok := t.(*task.RemoteTask); ok { - if b.Runtime.GetIgnoreErr() { - if len(b.Runtime.GetAllHosts()) > 0 { - if ac.GetStatus() == ending.FAILED { - res.Status = ending.SUCCESS - b.Runtime.DeleteHost(ac.Host) - } - } else { - result.ErrResult(errors.Wrapf(res.CombineErr(), "Module[%s] exec failed", b.Name)) - return - } - } - } - } - - if res.IsFailed() { - t.ExecuteRollback() - result.ErrResult(errors.Wrapf(res.CombineErr(), "Module[%s] exec failed", b.Name)) - return - } - } - result.NormalResult() -} diff --git a/cmd/kk/pkg/core/pipeline/pipeline.go b/cmd/kk/pkg/core/pipeline/pipeline.go deleted file mode 100644 index 9afce39d..00000000 --- a/cmd/kk/pkg/core/pipeline/pipeline.go +++ /dev/null @@ -1,164 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipeline - -import ( - "fmt" - "os" - "sync" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" -) - -var logo = ` - - _ __ _ _ __ -| | / / | | | | / / -| |/ / _ _| |__ ___| |/ / ___ _ _ -| \| | | | '_ \ / _ \ \ / _ \ | | | -| |\ \ |_| | |_) | __/ |\ \ __/ |_| | -\_| \_/\__,_|_.__/ \___\_| \_/\___|\__, | - __/ | - |___/ - -` - -type Pipeline struct { - Name string - Modules []module.Module - Runtime connector.Runtime - SpecHosts int - PipelineCache *cache.Cache - ModuleCachePool sync.Pool - ModulePostHooks []module.PostHookInterface -} - -func (p *Pipeline) Init() error { - fmt.Print(logo) - p.PipelineCache = cache.NewCache() - p.SpecHosts = len(p.Runtime.GetAllHosts()) - //if err := p.Runtime.GenerateWorkDir(); err != nil { - // return err - //} - //if err := p.Runtime.InitLogger(); err != nil { - // return err - //} - return nil -} - -func (p *Pipeline) Start() error { - if err := p.Init(); err != nil { - return errors.Wrapf(err, "Pipeline[%s] execute failed", p.Name) - } - for i := range p.Modules { - m := p.Modules[i] - if m.IsSkip() { - continue - } - - moduleCache := p.newModuleCache() - m.Default(p.Runtime, p.PipelineCache, moduleCache) - m.AutoAssert() - m.Init() - for j := range p.ModulePostHooks { - m.AppendPostHook(p.ModulePostHooks[j]) - } - - res := p.RunModule(m) - err := m.CallPostHook(res) - if res.IsFailed() { - return errors.Wrapf(res.CombineResult, "Pipeline[%s] execute failed", p.Name) - } - if err != nil { - return errors.Wrapf(err, "Pipeline[%s] execute failed", p.Name) - } - p.releaseModuleCache(moduleCache) - } - p.releasePipelineCache() - - // close ssh connect - for _, host := range p.Runtime.GetAllHosts() { - p.Runtime.GetConnector().Close(host) - } - - if p.SpecHosts != len(p.Runtime.GetAllHosts()) { - return errors.Errorf("Pipeline[%s] execute failed: there are some error in your spec hosts", p.Name) - } - logger.Log.Infof("Pipeline[%s] execute successfully", p.Name) - return nil -} - -func (p *Pipeline) RunModule(m module.Module) *ending.ModuleResult { - m.Slogan() - - result := ending.NewModuleResult() - for { - switch m.Is() { - case module.TaskModuleType: - m.Run(result) - if result.IsFailed() { - return result - } - - case module.GoroutineModuleType: - go func() { - m.Run(result) - if result.IsFailed() { - os.Exit(1) - } - }() - default: - m.Run(result) - if result.IsFailed() { - return result - } - } - - stop, err := m.Until() - if err != nil { - result.LocalErrResult(err) - return result - } - if stop == nil || *stop { - break - } - } - return result -} - -func (p *Pipeline) newModuleCache() *cache.Cache { - moduleCache, ok := p.ModuleCachePool.Get().(*cache.Cache) - if ok { - return moduleCache - } - return cache.NewCache() -} - -func (p *Pipeline) releasePipelineCache() { - p.PipelineCache.Clean() -} - -func (p *Pipeline) releaseModuleCache(c *cache.Cache) { - c.Clean() - p.ModuleCachePool.Put(c) -} diff --git a/cmd/kk/pkg/core/prepare/base.go b/cmd/kk/pkg/core/prepare/base.go deleted file mode 100644 index 46b644b0..00000000 --- a/cmd/kk/pkg/core/prepare/base.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package prepare - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type BasePrepare struct { - ModuleCache *cache.Cache - PipelineCache *cache.Cache -} - -func (b *BasePrepare) Init(moduleCache *cache.Cache, pipelineCache *cache.Cache) { - b.ModuleCache = moduleCache - b.PipelineCache = pipelineCache -} - -func (b *BasePrepare) PreCheck(runtime connector.Runtime) (bool, error) { - return true, nil -} - -func (b *BasePrepare) AutoAssert(runtime connector.Runtime) { -} - -type PrepareCollection []Prepare - -func (p *PrepareCollection) Init(cache *cache.Cache, rootCache *cache.Cache) { - for _, v := range *p { - v.Init(cache, rootCache) - } -} - -func (p *PrepareCollection) PreCheck(runtime connector.Runtime) (bool, error) { - for _, v := range *p { - res, err := v.PreCheck(runtime) - if err != nil { - return false, err - } - if res == false { - return false, nil - } - } - return true, nil -} - -func (p *PrepareCollection) AutoAssert(runtime connector.Runtime) { - for _, v := range *p { - v.AutoAssert(runtime) - } -} diff --git a/cmd/kk/pkg/core/prepare/fast_prepare.go b/cmd/kk/pkg/core/prepare/fast_prepare.go deleted file mode 100644 index 7f1566b8..00000000 --- a/cmd/kk/pkg/core/prepare/fast_prepare.go +++ /dev/null @@ -1,30 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package prepare - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type FastPrepare struct { - BasePrepare - Inject func(runtime connector.Runtime) (bool, error) -} - -func (b *FastPrepare) PreCheck(runtime connector.Runtime) (bool, error) { - return b.Inject(runtime) -} diff --git a/cmd/kk/pkg/core/prepare/file_check.go b/cmd/kk/pkg/core/prepare/file_check.go deleted file mode 100644 index b0cc21ed..00000000 --- a/cmd/kk/pkg/core/prepare/file_check.go +++ /dev/null @@ -1,38 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package prepare - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type FileExist struct { - BasePrepare - FilePath string - Not bool -} - -func (f *FileExist) PreCheck(runtime connector.Runtime) (bool, error) { - exist, err := runtime.GetRunner().FileExist(f.FilePath) - if err != nil { - return false, err - } - if f.Not { - return !exist, nil - } - return exist, nil -} diff --git a/cmd/kk/pkg/core/prepare/interface.go b/cmd/kk/pkg/core/prepare/interface.go deleted file mode 100644 index 46eb02dd..00000000 --- a/cmd/kk/pkg/core/prepare/interface.go +++ /dev/null @@ -1,28 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package prepare - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type Prepare interface { - PreCheck(runtime connector.Runtime) (bool, error) - Init(cache *cache.Cache, rootCache *cache.Cache) - AutoAssert(runtime connector.Runtime) -} diff --git a/cmd/kk/pkg/core/rollback/base.go b/cmd/kk/pkg/core/rollback/base.go deleted file mode 100644 index 838cd8ba..00000000 --- a/cmd/kk/pkg/core/rollback/base.go +++ /dev/null @@ -1,41 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package rollback - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" -) - -type BaseRollback struct { - ModuleCache *cache.Cache - PipelineCache *cache.Cache -} - -func (b *BaseRollback) Init(moduleCache *cache.Cache, pipelineCache *cache.Cache) { - b.ModuleCache = moduleCache - b.PipelineCache = pipelineCache -} - -func (b *BaseRollback) Execute(runtime connector.Runtime, result *ending.ActionResult) error { - return nil -} - -func (b *BaseRollback) AutoAssert(runtime connector.Runtime) { - -} diff --git a/cmd/kk/pkg/core/rollback/interface.go b/cmd/kk/pkg/core/rollback/interface.go deleted file mode 100644 index f0a28b23..00000000 --- a/cmd/kk/pkg/core/rollback/interface.go +++ /dev/null @@ -1,29 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package rollback - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" -) - -type Rollback interface { - Execute(runtime connector.Runtime, result *ending.ActionResult) (err error) - Init(cache *cache.Cache, rootCache *cache.Cache) - AutoAssert(runtime connector.Runtime) -} diff --git a/cmd/kk/pkg/core/task/const.go b/cmd/kk/pkg/core/task/const.go deleted file mode 100644 index eff13ee6..00000000 --- a/cmd/kk/pkg/core/task/const.go +++ /dev/null @@ -1,24 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package task - -const ( - DefaultTimeout = 120 - DefaultCon = 10 - - DefaultTaskName = "DefaultTask" -) diff --git a/cmd/kk/pkg/core/task/interface.go b/cmd/kk/pkg/core/task/interface.go deleted file mode 100644 index d1d154e7..00000000 --- a/cmd/kk/pkg/core/task/interface.go +++ /dev/null @@ -1,30 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package task - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" -) - -type Interface interface { - GetDesc() string - Init(runtime connector.Runtime, moduleCache *cache.Cache, pipelineCache *cache.Cache) - Execute() *ending.TaskResult - ExecuteRollback() -} diff --git a/cmd/kk/pkg/core/task/local_task.go b/cmd/kk/pkg/core/task/local_task.go deleted file mode 100644 index 0a197c64..00000000 --- a/cmd/kk/pkg/core/task/local_task.go +++ /dev/null @@ -1,231 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package task - -import ( - "context" - "fmt" - "time" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/rollback" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type LocalTask struct { - Name string - Desc string - Prepare prepare.Prepare - Action action.Action - Rollback rollback.Rollback - Retry int - Delay time.Duration - Timeout time.Duration - - PipelineCache *cache.Cache - ModuleCache *cache.Cache - Runtime connector.Runtime - tag string - IgnoreError bool - TaskResult *ending.TaskResult -} - -func (l *LocalTask) GetDesc() string { - return l.Desc -} - -func (l *LocalTask) Init(runtime connector.Runtime, moduleCache *cache.Cache, pipelineCache *cache.Cache) { - l.ModuleCache = moduleCache - l.PipelineCache = pipelineCache - l.Runtime = runtime - l.Default() -} - -func (l *LocalTask) Default() { - l.TaskResult = ending.NewTaskResult() - if l.Name == "" { - l.Name = DefaultTaskName - } - - if l.Prepare == nil { - l.Prepare = new(prepare.BasePrepare) - } - - if l.Action == nil { - l.TaskResult.AppendErr(nil, errors.New("the action is nil")) - return - } - - if l.Retry <= 0 { - l.Retry = 1 - } - - if l.Delay <= 0 { - l.Delay = 5 * time.Second - } - - if l.Timeout <= 0 { - l.Timeout = DefaultTimeout * time.Minute - } -} - -func (l *LocalTask) Execute() *ending.TaskResult { - if l.TaskResult.IsFailed() { - return l.TaskResult - } - - host := &connector.BaseHost{ - Name: common.LocalHost, - } - - selfRuntime := l.Runtime.Copy() - l.RunWithTimeout(selfRuntime, host) - - if l.TaskResult.IsFailed() { - l.TaskResult.ErrResult() - return l.TaskResult - } - - l.TaskResult.NormalResult() - return l.TaskResult -} - -func (l *LocalTask) RunWithTimeout(runtime connector.Runtime, host connector.Host) { - ctx, cancel := context.WithTimeout(context.Background(), l.Timeout) - defer cancel() - - resCh := make(chan error) - - go l.Run(runtime, host, resCh) - select { - case <-ctx.Done(): - l.TaskResult.AppendErr(host, fmt.Errorf("execute task timeout, Timeout=%s", util.ShortDur(l.Timeout))) - case e := <-resCh: - if e != nil { - l.TaskResult.AppendErr(host, e) - } - } -} - -func (l *LocalTask) Run(runtime connector.Runtime, host connector.Host, resCh chan error) { - var res error - defer func() { - resCh <- res - close(resCh) - }() - - runtime.SetRunner(&connector.Runner{ - Conn: nil, - //Debug: runtime.Arg.Debug, - Host: host, - }) - - l.Prepare.Init(l.ModuleCache, l.PipelineCache) - l.Prepare.AutoAssert(runtime) - if ok, err := l.WhenWithRetry(runtime, host); !ok { - if err != nil { - res = err - return - } else { - l.TaskResult.AppendSkip(host) - return - } - } - - l.Action.Init(l.ModuleCache, l.PipelineCache) - l.Action.AutoAssert(runtime) - if err := l.ExecuteWithRetry(runtime, host); err != nil { - res = err - return - } - l.TaskResult.AppendSuccess(host) -} - -func (l *LocalTask) WhenWithRetry(runtime connector.Runtime, host connector.Host) (bool, error) { - pass := false - err := fmt.Errorf("pre-check exec failed after %d retries", l.Retry) - for i := 0; i < l.Retry; i++ { - if res, e := l.When(runtime); e != nil { - logger.Log.Messagef(host.GetName(), e.Error()) - - if i == l.Retry-1 { - err = errors.New(err.Error() + e.Error()) - continue - } - logger.Log.Infof("retry: [%s]", host.GetName()) - time.Sleep(l.Delay) - continue - } else { - err = nil - pass = res - break - } - } - - return pass, err -} - -func (l *LocalTask) When(runtime connector.Runtime) (bool, error) { - if l.Prepare == nil { - return true, nil - } - if ok, err := l.Prepare.PreCheck(runtime); err != nil { - return false, err - } else if !ok { - return false, nil - } - return true, nil -} - -func (l *LocalTask) ExecuteWithRetry(runtime connector.Runtime, host connector.Host) error { - err := fmt.Errorf("[%s] exec failed after %d retries: ", l.Name, l.Retry) - for i := 0; i < l.Retry; i++ { - e := l.Action.Execute(runtime) - if e != nil { - logger.Log.Messagef(host.GetName(), e.Error()) - - if i == l.Retry-1 { - err = errors.New(err.Error() + e.Error()) - continue - } - logger.Log.Infof("retry: [%s]", host.GetName()) - time.Sleep(l.Delay) - continue - } else { - err = nil - break - } - } - return err -} - -func (l *LocalTask) ExecuteRollback() { - if l.Rollback == nil { - return - } - if !l.TaskResult.IsFailed() { - return - } -} diff --git a/cmd/kk/pkg/core/task/remote_task.go b/cmd/kk/pkg/core/task/remote_task.go deleted file mode 100644 index 4bb8c274..00000000 --- a/cmd/kk/pkg/core/task/remote_task.go +++ /dev/null @@ -1,347 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package task - -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/rollback" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type RemoteTask struct { - Name string - Desc string - Hosts []connector.Host - Prepare prepare.Prepare - Action action.Action - Rollback rollback.Rollback - Parallel bool - Retry int - Delay time.Duration - Timeout time.Duration - Concurrency float64 - - PipelineCache *cache.Cache - ModuleCache *cache.Cache - Runtime connector.Runtime - IgnoreError bool - TaskResult *ending.TaskResult -} - -func (t *RemoteTask) GetDesc() string { - return t.Desc -} - -func (t *RemoteTask) Init(runtime connector.Runtime, moduleCache *cache.Cache, pipelineCache *cache.Cache) { - t.ModuleCache = moduleCache - t.PipelineCache = pipelineCache - t.Runtime = runtime - t.Default() -} - -func (t *RemoteTask) Execute() *ending.TaskResult { - if t.TaskResult.IsFailed() { - return t.TaskResult - } - - routinePool := make(chan struct{}, DefaultCon) - defer close(routinePool) - - ctx, cancel := context.WithTimeout(context.Background(), t.Timeout) - defer cancel() - wg := &sync.WaitGroup{} - for i := range t.Hosts { - if t.Hosts[i] == nil || t.Runtime.HostIsDeprecated(t.Hosts[i]) { - continue - } - selfRuntime := t.Runtime.Copy() - - wg.Add(1) - if t.Parallel { - go t.RunWithTimeout(ctx, selfRuntime, t.Hosts[i], i, wg, routinePool) - } else { - t.RunWithTimeout(ctx, selfRuntime, t.Hosts[i], i, wg, routinePool) - } - } - wg.Wait() - - if t.TaskResult.IsFailed() { - t.TaskResult.ErrResult() - return t.TaskResult - } - - t.TaskResult.NormalResult() - return t.TaskResult -} - -func (t *RemoteTask) RunWithTimeout(ctx context.Context, runtime connector.Runtime, host connector.Host, index int, - wg *sync.WaitGroup, pool chan struct{}) { - - pool <- struct{}{} - - resCh := make(chan error) - go t.Run(runtime, host, index, resCh) - - select { - case <-ctx.Done(): - t.TaskResult.AppendErr(host, fmt.Errorf("execute task timeout, Timeout=%s", util.ShortDur(t.Timeout))) - case e := <-resCh: - if e != nil { - t.TaskResult.AppendErr(host, e) - } - } - - <-pool - wg.Done() -} - -func (t *RemoteTask) Run(runtime connector.Runtime, host connector.Host, index int, resCh chan error) { - var res error - defer func() { - //runtime.GetConnector().Close(host) - - resCh <- res - close(resCh) - }() - - if err := t.ConfigureSelfRuntime(runtime, host, index); err != nil { - res = err - return - } - - t.Prepare.Init(t.ModuleCache, t.PipelineCache) - t.Prepare.AutoAssert(runtime) - if ok, err := t.WhenWithRetry(runtime); !ok { - if err != nil { - res = err - return - } else { - t.TaskResult.AppendSkip(host) - return - } - } - - t.Action.Init(t.ModuleCache, t.PipelineCache) - t.Action.AutoAssert(runtime) - if err := t.ExecuteWithRetry(runtime); err != nil { - res = err - return - } - - t.TaskResult.AppendSuccess(host) - return -} - -func (t *RemoteTask) ConfigureSelfRuntime(runtime connector.Runtime, host connector.Host, index int) error { - conn, err := runtime.GetConnector().Connect(host) - if err != nil { - return errors.Wrapf(err, "failed to connect to %s", host.GetAddress()) - } - - r := &connector.Runner{ - Conn: conn, - //Debug: runtime.Arg.Debug, - Host: host, - Index: index, - } - runtime.SetRunner(r) - return nil -} - -func (t *RemoteTask) When(runtime connector.Runtime) (bool, error) { - if t.Prepare == nil { - return true, nil - } - if ok, err := t.Prepare.PreCheck(runtime); err != nil { - return false, err - } else if !ok { - return false, nil - } - return true, nil -} - -func (t *RemoteTask) WhenWithRetry(runtime connector.Runtime) (bool, error) { - pass := false - err := fmt.Errorf("pre-check exec failed after %d retries", t.Retry) - for i := 0; i < t.Retry; i++ { - if res, e := t.When(runtime); e != nil { - logger.Log.Messagef(runtime.RemoteHost().GetName(), e.Error()) - - if i == t.Retry-1 { - err = errors.New(err.Error() + e.Error()) - continue - } - logger.Log.Infof("retry: [%s]", runtime.GetRunner().Host.GetName()) - time.Sleep(t.Delay) - continue - } else { - err = nil - pass = res - break - } - } - - return pass, err -} - -func (t *RemoteTask) ExecuteWithRetry(runtime connector.Runtime) error { - err := fmt.Errorf("[%s] exec failed after %d retries: ", t.Name, t.Retry) - for i := 0; i < t.Retry; i++ { - e := t.Action.Execute(runtime) - if e != nil { - logger.Log.Messagef(runtime.RemoteHost().GetName(), e.Error()) - - if i == t.Retry-1 { - err = errors.New(err.Error() + e.Error()) - continue - } - logger.Log.Infof("retry: [%s]", runtime.GetRunner().Host.GetName()) - time.Sleep(t.Delay) - continue - } else { - err = nil - break - } - } - return err -} - -func (t *RemoteTask) ExecuteRollback() { - if t.Rollback == nil { - return - } - if !t.TaskResult.IsFailed() { - return - } - - ctx, cancel := context.WithTimeout(context.Background(), t.Timeout) - defer cancel() - routinePool := make(chan struct{}, DefaultCon) - defer close(routinePool) - - rwg := &sync.WaitGroup{} - for i, ar := range t.TaskResult.ActionResults { - if ar.Host == nil || t.Runtime.HostIsDeprecated(ar.Host) { - continue - } - selfRuntime := t.Runtime.Copy() - - if t.Parallel { - go t.RollbackWithTimeout(ctx, selfRuntime, ar.Host, i, ar, rwg, routinePool) - } else { - t.RollbackWithTimeout(ctx, selfRuntime, ar.Host, i, ar, rwg, routinePool) - } - rwg.Add(1) - } - rwg.Wait() -} - -func (t *RemoteTask) RollbackWithTimeout(ctx context.Context, runtime connector.Runtime, host connector.Host, index int, - result *ending.ActionResult, wg *sync.WaitGroup, pool chan struct{}) { - - pool <- struct{}{} - - resCh := make(chan error) - go t.RunRollback(runtime, host, index, result, resCh) - - select { - case <-ctx.Done(): - logger.Log.Errorf("rollback-failed: [%s]", runtime.GetRunner().Host.GetName()) - logger.Log.Messagef(runtime.RemoteHost().GetName(), fmt.Sprintf("execute task timeout, Timeout=%d", t.Timeout)) - case e := <-resCh: - if e != nil { - logger.Log.Errorf("rollback-failed: [%s]", runtime.GetRunner().Host.GetName()) - logger.Log.Messagef(runtime.RemoteHost().GetName(), e.Error()) - } - } - - <-pool - wg.Done() -} - -func (t *RemoteTask) RunRollback(runtime connector.Runtime, host connector.Host, index int, result *ending.ActionResult, resCh chan error) { - var res error - defer func() { - //runtime.GetConnector().Close(host) - - resCh <- res - close(resCh) - }() - - if err := t.ConfigureSelfRuntime(runtime, host, index); err != nil { - res = err - return - } - - logger.Log.Infof("rollback: [%s]", runtime.GetRunner().Host.GetName()) - - t.Rollback.Init(t.ModuleCache, t.PipelineCache) - t.Rollback.AutoAssert(runtime) - if err := t.Rollback.Execute(runtime, result); err != nil { - res = err - return - } - return -} - -func (t *RemoteTask) Default() { - t.TaskResult = ending.NewTaskResult() - if t.Name == "" { - t.Name = DefaultTaskName - } - - if t.Prepare == nil { - t.Prepare = new(prepare.BasePrepare) - } - - if t.Retry <= 0 { - t.Retry = 3 - } - - if t.Delay <= 0 { - t.Delay = 5 * time.Second - } - - if t.Timeout <= 0 { - t.Timeout = DefaultTimeout * time.Minute - } - - if t.Concurrency <= 0 || t.Concurrency > 1 { - t.Concurrency = 1 - } -} - -func (t *RemoteTask) calculateConcurrency() int { - num := t.Concurrency * float64(len(t.Hosts)) - res := int(util.Round(num, 0)) - if res < 1 { - res = 1 - } - return res -} diff --git a/cmd/kk/pkg/core/task/remote_task_test.go b/cmd/kk/pkg/core/task/remote_task_test.go deleted file mode 100644 index ad80f9d2..00000000 --- a/cmd/kk/pkg/core/task/remote_task_test.go +++ /dev/null @@ -1,141 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package task - -import ( - "testing" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -func TestTask_calculateConcurrency(t1 *testing.T) { - type fields struct { - Hosts []connector.BaseHost - Concurrency float64 - } - tests := []struct { - name string - fields fields - want int - }{ - { - name: "test1", - fields: fields{ - Concurrency: 0.5, - Hosts: []connector.BaseHost{ - {Name: "node1"}, - {Name: "node2"}, - {Name: "node3"}, - }, - }, - want: 2, - }, - { - name: "test2", - fields: fields{ - Concurrency: 0.5, - Hosts: []connector.BaseHost{ - {Name: "node1"}, - {Name: "node2"}, - {Name: "node3"}, - {Name: "node4"}, - }, - }, - want: 2, - }, - { - name: "test3", - fields: fields{ - Concurrency: 0.4, - Hosts: []connector.BaseHost{ - {Name: "node1"}, - {Name: "node2"}, - {Name: "node3"}, - }, - }, - want: 1, - }, - { - name: "test4", - fields: fields{ - Concurrency: 0.4, - Hosts: []connector.BaseHost{ - {Name: "node1"}, - {Name: "node2"}, - {Name: "node3"}, - {Name: "node4"}, - }, - }, - want: 2, - }, - { - name: "test5", - fields: fields{ - Concurrency: 0.1, - Hosts: []connector.BaseHost{ - {Name: "node1"}, - {Name: "node2"}, - {Name: "node3"}, - {Name: "node4"}, - }, - }, - want: 1, - }, - { - name: "test6", - fields: fields{ - Concurrency: 0.222222222222222, - Hosts: []connector.BaseHost{ - {Name: "node1"}, - {Name: "node2"}, - {Name: "node3"}, - {Name: "node4"}, - }, - }, - want: 1, - }, - { - name: "test7", - fields: fields{ - Concurrency: 1, - Hosts: []connector.BaseHost{ - {Name: "node1"}, - {Name: "node2"}, - {Name: "node3"}, - {Name: "node4"}, - }, - }, - want: 4, - }, - } - for _, tt := range tests { - t1.Run(tt.name, func(t1 *testing.T) { - var hosts []connector.Host - for _, v := range tt.fields.Hosts { - hosts = append(hosts, &v) - } - - t := &RemoteTask{ - Concurrency: tt.fields.Concurrency, - Hosts: hosts, - } - if got := t.calculateConcurrency(); got != tt.want { - t1.Errorf("calculateConcurrency() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/cmd/kk/pkg/core/util/file.go b/cmd/kk/pkg/core/util/file.go deleted file mode 100644 index cdb3e36c..00000000 --- a/cmd/kk/pkg/core/util/file.go +++ /dev/null @@ -1,243 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import ( - "archive/tar" - "compress/gzip" - "crypto/md5" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "strings" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -func IsExist(path string) bool { - _, err := os.Stat(path) - if err != nil { - if os.IsExist(err) { - return true - } - if os.IsNotExist(err) { - return false - } - return false - } - return true -} - -func CreateDir(path string) error { - if IsExist(path) == false { - err := os.MkdirAll(path, os.ModePerm) - if err != nil { - return err - } - } - return nil -} - -func IsDir(path string) bool { - s, err := os.Stat(path) - if err != nil { - return false - } - return s.IsDir() -} - -func CountDirFiles(dirName string) int { - if !IsDir(dirName) { - return 0 - } - var count int - err := filepath.Walk(dirName, func(path string, info os.FileInfo, err error) error { - if info.IsDir() { - return nil - } - count++ - return nil - }) - if err != nil { - logger.Log.Fatalln("count dir files failed %v", err) - return 0 - } - return count -} - -// FileMD5 count file md5 -func FileMD5(path string) (string, error) { - file, err := os.Open(path) - if err != nil { - return "", err - } - - m := md5.New() - if _, err := io.Copy(m, file); err != nil { - return "", err - } - - fileMd5 := fmt.Sprintf("%x", m.Sum(nil)) - return fileMd5, nil -} - -func LocalMd5Sum(src string) string { - md5Str, err := FileMD5(src) - if err != nil { - logger.Log.Fatalf("get file md5 failed %v", err) - return "" - } - return md5Str -} - -// MkFileFullPathDir is used to file create the full path. -// eg. there is a file "./aa/bb/xxx.txt", and dir ./aa/bb is not exist, and will create the full path dir. -func MkFileFullPathDir(fileName string) error { - localDir := filepath.Dir(fileName) - err := Mkdir(localDir) - if err != nil { - return fmt.Errorf("create local dir %s failed: %v", localDir, err) - } - return nil -} - -func Mkdir(dirName string) error { - return os.MkdirAll(dirName, os.ModePerm) -} - -func WriteFile(fileName string, content []byte) error { - dir := filepath.Dir(fileName) - if _, err := os.Stat(dir); os.IsNotExist(err) { - if err = os.MkdirAll(dir, common.FileMode0755); err != nil { - return err - } - } - - if err := os.WriteFile(fileName, content, common.FileMode0644); err != nil { - return err - } - return nil -} - -func Tar(src, dst, trimPrefix string) error { - fw, err := os.Create(dst) - if err != nil { - return err - } - defer fw.Close() - - gw := gzip.NewWriter(fw) - defer gw.Close() - - tw := tar.NewWriter(gw) - defer tw.Close() - - return filepath.Walk(src, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - - hdr, err := tar.FileInfoHeader(info, "") - if err != nil { - return err - } - - if !info.Mode().IsRegular() { - return nil - } - - fr, err := os.Open(path) - defer fr.Close() - if err != nil { - return err - } - - path = strings.TrimPrefix(path, trimPrefix) - fmt.Println(strings.TrimPrefix(path, string(filepath.Separator))) - - hdr.Name = strings.TrimPrefix(path, string(filepath.Separator)) - if err := tw.WriteHeader(hdr); err != nil { - return err - } - - if _, err := io.Copy(tw, fr); err != nil { - return err - } - - return nil - }) -} - -func Untar(src, dst string) error { - fr, err := os.Open(src) - if err != nil { - return err - } - defer fr.Close() - - gr, err := gzip.NewReader(fr) - if err != nil { - return err - } - defer gr.Close() - - tr := tar.NewReader(gr) - for { - hdr, err := tr.Next() - - switch { - case err == io.EOF: - return nil - case err != nil: - return err - case hdr == nil: - continue - } - - dstPath := filepath.Join(dst, hdr.Name) - - switch hdr.Typeflag { - case tar.TypeDir: - if !IsExist(dstPath) && IsDir(dstPath) { - if err := CreateDir(dstPath); err != nil { - return err - } - } - case tar.TypeReg: - if dir := filepath.Dir(dstPath); !IsExist(dir) { - if err := CreateDir(dir); err != nil { - return err - } - } - - file, err := os.OpenFile(dstPath, os.O_CREATE|os.O_RDWR, os.FileMode(hdr.Mode)) - if err != nil { - return err - } - - if _, err = io.Copy(file, tr); err != nil { - return err - } - - fmt.Println(dstPath) - file.Close() - } - } -} diff --git a/cmd/kk/pkg/core/util/ip.go b/cmd/kk/pkg/core/util/ip.go deleted file mode 100644 index 162650a8..00000000 --- a/cmd/kk/pkg/core/util/ip.go +++ /dev/null @@ -1,168 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import ( - "encoding/binary" - "net" - "strconv" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -func ParseIp(ip string) []string { - var availableIPs []string - // if ip is "1.1.1.1/",trim / - ip = strings.TrimRight(ip, "/") - if strings.Contains(ip, "/") == true { - if strings.Contains(ip, "/32") == true { - aip := strings.Replace(ip, "/32", "", -1) - availableIPs = append(availableIPs, aip) - } else { - availableIPs = GetAvailableIP(ip) - } - } else if strings.Contains(ip, "-") == true { - ipRange := strings.SplitN(ip, "-", 2) - availableIPs = GetAvailableIPRange(ipRange[0], ipRange[1]) - } else { - availableIPs = append(availableIPs, ip) - } - return availableIPs -} - -func GetAvailableIPRange(ipStart, ipEnd string) []string { - var availableIPs []string - - firstIP := net.ParseIP(ipStart) - endIP := net.ParseIP(ipEnd) - if firstIP.To4() == nil || endIP.To4() == nil { - return availableIPs - } - firstIPNum := ipToInt(firstIP.To4()) - EndIPNum := ipToInt(endIP.To4()) - pos := int32(1) - - newNum := firstIPNum - - for newNum <= EndIPNum { - availableIPs = append(availableIPs, intToIP(newNum).String()) - newNum = newNum + pos - } - return availableIPs -} - -func GetAvailableIP(ipAndMask string) []string { - var availableIPs []string - - ipAndMask = strings.TrimSpace(ipAndMask) - ipAndMask = IPAddressToCIDR(ipAndMask) - _, ipnet, _ := net.ParseCIDR(ipAndMask) - - firstIP, _ := networkRange(ipnet) - ipNum := ipToInt(firstIP) - size := networkSize(ipnet.Mask) - pos := int32(1) - max := size - 2 // -1 for the broadcast address, -1 for the gateway address - - var newNum int32 - for attempt := int32(0); attempt < max; attempt++ { - newNum = ipNum + pos - pos = pos%max + 1 - availableIPs = append(availableIPs, intToIP(newNum).String()) - } - return availableIPs -} - -func IPAddressToCIDR(ipAddress string) string { - if strings.Contains(ipAddress, "/") == true { - ipAndMask := strings.Split(ipAddress, "/") - ip := ipAndMask[0] - mask := ipAndMask[1] - if strings.Contains(mask, ".") == true { - mask = IPMaskStringToCIDR(mask) - } - return ip + "/" + mask - } else { - return ipAddress - } -} - -func IPMaskStringToCIDR(netmask string) string { - netmaskList := strings.Split(netmask, ".") - var mint []int - for _, v := range netmaskList { - strv, _ := strconv.Atoi(v) - mint = append(mint, strv) - } - myIPMask := net.IPv4Mask(byte(mint[0]), byte(mint[1]), byte(mint[2]), byte(mint[3])) - ones, _ := myIPMask.Size() - return strconv.Itoa(ones) -} - -func networkRange(network *net.IPNet) (net.IP, net.IP) { - netIP := network.IP.To4() - firstIP := netIP.Mask(network.Mask) - lastIP := net.IPv4(0, 0, 0, 0).To4() - for i := 0; i < len(lastIP); i++ { - lastIP[i] = netIP[i] | ^network.Mask[i] - } - return firstIP, lastIP -} - -func networkSize(mask net.IPMask) int32 { - m := net.IPv4Mask(0, 0, 0, 0) - for i := 0; i < net.IPv4len; i++ { - m[i] = ^mask[i] - } - return int32(binary.BigEndian.Uint32(m)) + 1 -} - -func ipToInt(ip net.IP) int32 { - return int32(binary.BigEndian.Uint32(ip.To4())) -} - -func intToIP(n int32) net.IP { - b := make([]byte, 4) - binary.BigEndian.PutUint32(b, uint32(n)) - return net.IP(b) -} - -func GetLocalIP() (string, error) { - addrs, err := net.InterfaceAddrs() - if err != nil { - return "", err - } - for _, addr := range addrs { - if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { - if ipnet.IP.To4() != nil && ipnet.IP.IsGlobalUnicast() { - return ipnet.IP.String(), nil - } - } - } - return "", errors.New("valid local IP not found!") -} - -func LocalIP() string { - localIp, err := GetLocalIP() - if err != nil { - logger.Log.Fatalf("Failed to get Local IP: %v", err) - } - return localIp -} diff --git a/cmd/kk/pkg/core/util/time.go b/cmd/kk/pkg/core/util/time.go deleted file mode 100644 index 5f1d137f..00000000 --- a/cmd/kk/pkg/core/util/time.go +++ /dev/null @@ -1,33 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import ( - "strings" - "time" -) - -func ShortDur(d time.Duration) string { - s := d.String() - if strings.HasSuffix(s, "m0s") { - s = s[:len(s)-2] - } - if strings.HasSuffix(s, "h0m") { - s = s[:len(s)-2] - } - return s -} diff --git a/cmd/kk/pkg/core/util/time_test.go b/cmd/kk/pkg/core/util/time_test.go deleted file mode 100644 index 7a18073b..00000000 --- a/cmd/kk/pkg/core/util/time_test.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import ( - "testing" - "time" -) - -func TestShortDur(t *testing.T) { - tests := []struct { - d time.Duration - want string - }{ - { - d: 3 * time.Second, - want: "3s", - }, - { - d: 60 * time.Second, - want: "1m", - }, - { - d: 4 * time.Minute, - want: "4m", - }, - { - d: 1*time.Hour + 3*time.Second, - want: "1h0m3s", - }, - { - d: 1*time.Hour + 4*time.Minute, - want: "1h4m", - }, - { - d: 1*time.Hour + 4*time.Minute + 3*time.Second, - want: "1h4m3s", - }, - } - for _, tt := range tests { - t.Run("", func(t *testing.T) { - if got := ShortDur(tt.d); got != tt.want { - t.Errorf("ShortDur() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/cmd/kk/pkg/core/util/util.go b/cmd/kk/pkg/core/util/util.go deleted file mode 100644 index d14c0f6f..00000000 --- a/cmd/kk/pkg/core/util/util.go +++ /dev/null @@ -1,138 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import ( - "bytes" - "fmt" - "math" - "os" - "os/exec" - "os/user" - "runtime" - "sort" - "strings" - "text/template" - - "github.com/pkg/errors" -) - -type Data map[string]interface{} - -// Render text template with given `variables` Render-context -func Render(tmpl *template.Template, variables map[string]interface{}) (string, error) { - - var buf strings.Builder - - if err := tmpl.Execute(&buf, variables); err != nil { - return "", errors.Wrap(err, "Failed to render template") - } - return buf.String(), nil -} - -// Home returns the home directory for the executing user. -func Home() (string, error) { - u, err := user.Current() - if nil == err { - return u.HomeDir, nil - } - - if "windows" == runtime.GOOS { - return homeWindows() - } - - return homeUnix() -} - -func homeUnix() (string, error) { - if home := os.Getenv("HOME"); home != "" { - return home, nil - } - - var stdout bytes.Buffer - cmd := exec.Command("sh", "-c", "eval echo ~$USER") - cmd.Stdout = &stdout - if err := cmd.Run(); err != nil { - return "", err - } - - result := strings.TrimSpace(stdout.String()) - if result == "" { - return "", errors.New("blank output when reading home directory") - } - - return result, nil -} - -func homeWindows() (string, error) { - drive := os.Getenv("HOMEDRIVE") - path := os.Getenv("HOMEPATH") - home := drive + path - if drive == "" || path == "" { - home = os.Getenv("USERPROFILE") - } - if home == "" { - return "", errors.New("HOMEDRIVE, HOMEPATH, and USERPROFILE are blank") - } - - return home, nil -} - -func GetArgs(argsMap map[string]string, args []string) ([]string, map[string]string) { - targetMap := make(map[string]string, len(argsMap)) - for k, v := range argsMap { - targetMap[k] = v - } - targetSlice := make([]string, len(args)) - copy(targetSlice, args) - - for _, arg := range targetSlice { - splitArg := strings.SplitN(arg, "=", 2) - if len(splitArg) < 2 { - continue - } - targetMap[splitArg[0]] = splitArg[1] - } - - for arg, value := range targetMap { - cmd := fmt.Sprintf("%s=%s", arg, value) - targetSlice = append(targetSlice, cmd) - } - sort.Strings(targetSlice) - return targetSlice, targetMap -} - -// Round returns the result of rounding 'val' according to the specified 'precision' precision (the number of digits after the decimal point)。 -// and precision can be negative number or zero -func Round(val float64, precision int) float64 { - p := math.Pow10(precision) - return math.Floor(val*p+0.5) / p -} - -// ArchAlias returns the alias of cpu's architecture. -// amd64: x86_64 -// arm64: aarch64 -func ArchAlias(arch string) string { - switch arch { - case "amd64": - return "x86_64" - case "arm64": - return "aarch64" - default: - return "" - } -} diff --git a/cmd/kk/pkg/core/util/util_test.go b/cmd/kk/pkg/core/util/util_test.go deleted file mode 100644 index 8351d73e..00000000 --- a/cmd/kk/pkg/core/util/util_test.go +++ /dev/null @@ -1,55 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import "testing" - -func TestRound(t *testing.T) { - type args struct { - val float64 - precision int - } - tests := []struct { - name string - args args - want float64 - }{ - { - name: "test1", - args: args{ - val: 0.5555, - precision: 0, - }, - want: 1, - }, - { - name: "test2", - args: args{ - val: 0.4555, - precision: 0, - }, - want: 0, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := Round(tt.args.val, tt.args.precision); got != tt.want { - t.Errorf("Round() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/cmd/kk/pkg/etcd/certs.go b/cmd/kk/pkg/etcd/certs.go deleted file mode 100644 index d481ae1d..00000000 --- a/cmd/kk/pkg/etcd/certs.go +++ /dev/null @@ -1,262 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package etcd - -import ( - "crypto/x509" - "fmt" - "net" - "os" - "path/filepath" - "strings" - - "github.com/pkg/errors" - "k8s.io/client-go/util/cert" - certutil "k8s.io/client-go/util/cert" - netutils "k8s.io/utils/net" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils/certs" -) - -// KubekeyCertEtcdCA is the definition of the root CA used by the hosted etcd server. -func KubekeyCertEtcdCA() *certs.KubekeyCert { - return &certs.KubekeyCert{ - Name: "etcd-ca", - LongName: "self-signed CA to provision identities for etcd", - BaseName: "ca", - Config: certs.CertConfig{ - Config: certutil.Config{ - CommonName: "etcd-ca", - }, - }, - } -} - -// KubekeyCertEtcdAdmin is the definition of the cert for etcd admin. -func KubekeyCertEtcdAdmin(hostname string, altNames *certutil.AltNames) *certs.KubekeyCert { - l := strings.Split(hostname, ".") - return &certs.KubekeyCert{ - Name: "etcd-admin", - LongName: "certificate for etcd admin", - BaseName: fmt.Sprintf("admin-%s", hostname), - CAName: "etcd-ca", - Config: certs.CertConfig{ - Config: certutil.Config{ - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - AltNames: *altNames, - CommonName: fmt.Sprintf("etcd-admin-%s", l[0]), - }, - }, - } -} - -// KubekeyCertEtcdMember is the definition of the cert for etcd member. -func KubekeyCertEtcdMember(hostname string, altNames *certutil.AltNames) *certs.KubekeyCert { - l := strings.Split(hostname, ".") - return &certs.KubekeyCert{ - Name: "etcd-member", - LongName: "certificate for etcd member", - BaseName: fmt.Sprintf("member-%s", hostname), - CAName: "etcd-ca", - Config: certs.CertConfig{ - Config: certutil.Config{ - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - AltNames: *altNames, - CommonName: fmt.Sprintf("etcd-member-%s", l[0]), - }, - }, - } -} - -// KubekeyCertEtcdClient is the definition of the cert for etcd client. -func KubekeyCertEtcdClient(hostname string, altNames *certutil.AltNames) *certs.KubekeyCert { - l := strings.Split(hostname, ".") - return &certs.KubekeyCert{ - Name: "etcd-client", - LongName: "certificate for etcd client", - BaseName: fmt.Sprintf("node-%s", hostname), - CAName: "etcd-ca", - Config: certs.CertConfig{ - Config: certutil.Config{ - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - AltNames: *altNames, - CommonName: fmt.Sprintf("etcd-node-%s", l[0]), - }, - }, - } -} - -type FetchCerts struct { - common.KubeAction -} - -func (f *FetchCerts) Execute(runtime connector.Runtime) error { - src := "/etc/ssl/etcd/ssl" - dst := fmt.Sprintf("%s/pki/etcd", runtime.GetWorkDir()) - - v, ok := f.PipelineCache.Get(common.ETCDCluster) - if !ok { - return errors.New("get etcd status from pipeline cache failed") - } - - c := v.(*EtcdCluster) - - if c.clusterExist { - certs, err := runtime.GetRunner().SudoCmd("ls /etc/ssl/etcd/ssl/ | grep .pem", false) - if err != nil { - return errors.Wrap(err, "failed to find certificate files") - } - - certsList := strings.Split(certs, "\r\n") - if len(certsList) > 0 { - for _, cert := range certsList { - if err := runtime.GetRunner().Fetch(filepath.Join(dst, cert), filepath.Join(src, cert)); err != nil { - return errors.Wrap(err, fmt.Sprintf("Fetch %s failed", filepath.Join(src, cert))) - } - } - } - } - - return nil -} - -type GenerateCerts struct { - common.KubeAction -} - -func (g *GenerateCerts) Execute(runtime connector.Runtime) error { - - pkiPath := fmt.Sprintf("%s/pki/etcd", runtime.GetWorkDir()) - - altName := GenerateAltName(g.KubeConf, &runtime) - - files := []string{"ca.pem", "ca-key.pem"} - - // CA - certsList := []*certs.KubekeyCert{KubekeyCertEtcdCA()} - - // Certs - for _, host := range runtime.GetAllHosts() { - if host.IsRole(common.ETCD) { - certsList = append(certsList, KubekeyCertEtcdAdmin(host.GetName(), altName)) - files = append(files, []string{fmt.Sprintf("admin-%s.pem", host.GetName()), fmt.Sprintf("admin-%s-key.pem", host.GetName())}...) - certsList = append(certsList, KubekeyCertEtcdMember(host.GetName(), altName)) - files = append(files, []string{fmt.Sprintf("member-%s.pem", host.GetName()), fmt.Sprintf("member-%s-key.pem", host.GetName())}...) - } - if host.IsRole(common.Master) { - certsList = append(certsList, KubekeyCertEtcdClient(host.GetName(), altName)) - files = append(files, []string{fmt.Sprintf("node-%s.pem", host.GetName()), fmt.Sprintf("node-%s-key.pem", host.GetName())}...) - } - } - - var lastCACert *certs.KubekeyCert - for _, c := range certsList { - if c.CAName == "" { - err := certs.GenerateCA(c, pkiPath, g.KubeConf) - if err != nil { - return err - } - lastCACert = c - } else { - err := certs.GenerateCerts(c, lastCACert, pkiPath, g.KubeConf) - if err != nil { - return err - } - } - } - - g.ModuleCache.Set(LocalCertsDir, pkiPath) - g.ModuleCache.Set(CertsFileList, files) - - return nil -} - -func GenerateAltName(k *common.KubeConf, runtime *connector.Runtime) *cert.AltNames { - var altName cert.AltNames - - dnsList := []string{"localhost", "etcd.kube-system.svc.cluster.local", "etcd.kube-system.svc", "etcd.kube-system", "etcd"} - ipList := []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback} - - if k.Cluster.ControlPlaneEndpoint.Domain == "" { - dnsList = append(dnsList, kubekeyapiv1alpha2.DefaultLBDomain) - } else { - dnsList = append(dnsList, k.Cluster.ControlPlaneEndpoint.Domain) - } - - for _, host := range k.Cluster.Hosts { - dnsList = append(dnsList, host.Name) - internalAddress := netutils.ParseIPSloppy(host.InternalAddress) - if internalAddress != nil { - ipList = append(ipList, internalAddress) - } - } - - altName.DNSNames = dnsList - altName.IPs = ipList - - return &altName -} - -type FetchCertsForExternalEtcd struct { - common.KubeAction -} - -func (f *FetchCertsForExternalEtcd) Execute(runtime connector.Runtime) error { - - pkiPath := fmt.Sprintf("%s/pki/etcd", runtime.GetWorkDir()) - - if err := util.CreateDir(pkiPath); err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to create dir %s", pkiPath)) - } - - srcCertsFiles := []string{f.KubeConf.Cluster.Etcd.External.CAFile, f.KubeConf.Cluster.Etcd.External.CertFile, f.KubeConf.Cluster.Etcd.External.KeyFile} - dstCertsFiles := []string{} - for _, certFile := range srcCertsFiles { - if len(certFile) != 0 { - certPath, err := filepath.Abs(certFile) - if err != nil { - return errors.Wrap(err, "bad certificate file path") - } - _, err = os.Stat(certPath) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("%s does not exist", certPath)) - } - - dstCertFileName := filepath.Base(certPath) - dstCert := fmt.Sprintf("%s/%s", pkiPath, dstCertFileName) - dstCertsFiles = append(dstCertsFiles, dstCertFileName) - - data, err := os.ReadFile(certPath) - if err != nil { - return errors.Wrap(err, "failed to copy certificate content") - } - - if err := os.WriteFile(dstCert, data, 0600); err != nil { - return errors.Wrap(err, "failed to copy certificate content") - } - } - } - - f.ModuleCache.Set(LocalCertsDir, pkiPath) - f.ModuleCache.Set(CertsFileList, dstCertsFiles) - - return nil -} diff --git a/cmd/kk/pkg/etcd/module.go b/cmd/kk/pkg/etcd/module.go deleted file mode 100644 index f454cba0..00000000 --- a/cmd/kk/pkg/etcd/module.go +++ /dev/null @@ -1,425 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package etcd - -import ( - "path/filepath" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/etcd/templates" -) - -type PreCheckModule struct { - common.KubeModule - Skip bool -} - -func (p *PreCheckModule) IsSkip() bool { - return p.Skip -} - -func (p *PreCheckModule) Init() { - p.Name = "ETCDPreCheckModule" - p.Desc = "Get ETCD cluster status" - - getStatus := &task.RemoteTask{ - Name: "GetETCDStatus", - Desc: "Get etcd status", - Hosts: p.Runtime.GetHostsByRole(common.ETCD), - Action: new(GetStatus), - Parallel: false, - Retry: 0, - } - p.Tasks = []task.Interface{ - getStatus, - } -} - -type CertsModule struct { - common.KubeModule - Skip bool -} - -func (p *CertsModule) IsSkip() bool { - return p.Skip -} - -func (c *CertsModule) Init() { - c.Name = "CertsModule" - c.Desc = "Sign ETCD cluster certs" - - switch c.KubeConf.Cluster.Etcd.Type { - case kubekeyapiv1alpha2.KubeKey: - c.Tasks = CertsModuleForKubeKey(c) - case kubekeyapiv1alpha2.External: - c.Tasks = CertsModuleForExternal(c) - } -} - -func CertsModuleForKubeKey(c *CertsModule) []task.Interface { - // If the etcd cluster already exists, obtain the certificate in use from the etcd node. - fetchCerts := &task.RemoteTask{ - Name: "FetchETCDCerts", - Desc: "Fetch etcd certs", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: new(FirstETCDNode), - Action: new(FetchCerts), - Parallel: false, - } - - generateCerts := &task.LocalTask{ - Name: "GenerateETCDCerts", - Desc: "Generate etcd Certs", - Action: new(GenerateCerts), - } - - syncCertsFile := &task.RemoteTask{ - Name: "SyncCertsFile", - Desc: "Synchronize certs file", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Action: new(SyncCertsFile), - Parallel: true, - Retry: 1, - } - - syncCertsToMaster := &task.RemoteTask{ - Name: "SyncCertsFileToMaster", - Desc: "Synchronize certs file to master", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &common.OnlyETCD{Not: true}, - Action: new(SyncCertsFile), - Parallel: true, - Retry: 1, - } - - return []task.Interface{ - fetchCerts, - generateCerts, - syncCertsFile, - syncCertsToMaster, - } -} - -func CertsModuleForExternal(c *CertsModule) []task.Interface { - fetchCerts := &task.LocalTask{ - Name: "FetchETCDCerts", - Desc: "Fetch etcd certs", - Action: new(FetchCertsForExternalEtcd), - } - - syncCertsToMaster := &task.RemoteTask{ - Name: "SyncCertsFileToMaster", - Desc: "Synchronize certs file to master", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Action: new(SyncCertsFile), - Parallel: true, - Retry: 1, - } - - return []task.Interface{ - fetchCerts, - syncCertsToMaster, - } -} - -type InstallETCDBinaryModule struct { - common.KubeModule - Skip bool -} - -func (p *InstallETCDBinaryModule) IsSkip() bool { - return p.Skip -} - -func (i *InstallETCDBinaryModule) Init() { - i.Name = "InstallETCDBinaryModule" - i.Desc = "Install ETCD cluster" - - installETCDBinary := &task.RemoteTask{ - Name: "InstallETCDBinary", - Desc: "Install etcd using binary", - Hosts: i.Runtime.GetHostsByRole(common.ETCD), - Action: new(InstallETCDBinary), - Parallel: true, - Retry: 1, - } - - generateETCDService := &task.RemoteTask{ - Name: "GenerateETCDService", - Desc: "Generate etcd service", - Hosts: i.Runtime.GetHostsByRole(common.ETCD), - Action: &action.Template{ - Template: templates.ETCDService, - Dst: "/etc/systemd/system/etcd.service", - }, - Parallel: true, - Retry: 1, - } - - accessAddress := &task.RemoteTask{ - Name: "GenerateAccessAddress", - Desc: "Generate access address", - Hosts: i.Runtime.GetHostsByRole(common.ETCD), - Prepare: new(FirstETCDNode), - Action: new(GenerateAccessAddress), - Parallel: true, - Retry: 1, - } - - i.Tasks = []task.Interface{ - installETCDBinary, - generateETCDService, - accessAddress, - } -} - -type ConfigureModule struct { - common.KubeModule - Skip bool -} - -func (p *ConfigureModule) IsSkip() bool { - return p.Skip -} - -func (e *ConfigureModule) Init() { - e.Name = "ETCDConfigureModule" - e.Desc = "Configure ETCD cluster" - - if v, ok := e.PipelineCache.Get(common.ETCDCluster); ok { - cluster := v.(*EtcdCluster) - if !cluster.clusterExist { - e.Tasks = handleNewCluster(e) - } else { - e.Tasks = handleExistCluster(e) - } - } -} - -func handleNewCluster(c *ConfigureModule) []task.Interface { - - existETCDHealthCheck := &task.RemoteTask{ - Name: "ExistETCDHealthCheck", - Desc: "Health check on exist etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: new(NodeETCDExist), - Action: new(HealthCheck), - Parallel: true, - Retry: 20, - } - - generateETCDConfig := &task.RemoteTask{ - Name: "GenerateETCDConfig", - Desc: "Generate etcd.env config on new etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: &NodeETCDExist{Not: true}, - Action: new(GenerateConfig), - Parallel: false, - } - - allRefreshETCDConfig := &task.RemoteTask{ - Name: "AllRefreshETCDConfig", - Desc: "Refresh etcd.env config on all etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Action: new(RefreshConfig), - Parallel: false, - } - - restart := &task.RemoteTask{ - Name: "RestartETCD", - Desc: "Restart etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: &NodeETCDExist{Not: true}, - Action: new(RestartETCD), - Parallel: true, - } - - allETCDNodeHealthCheck := &task.RemoteTask{ - Name: "AllETCDNodeHealthCheck", - Desc: "Health check on all etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Action: new(HealthCheck), - Parallel: true, - Retry: 20, - } - - refreshETCDConfigToExist := &task.RemoteTask{ - Name: "RefreshETCDConfigToExist", - Desc: "Refresh etcd.env config to exist mode on all etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Action: &RefreshConfig{ToExisting: true}, - Parallel: false, - } - - tasks := []task.Interface{ - existETCDHealthCheck, - generateETCDConfig, - allRefreshETCDConfig, - restart, - allETCDNodeHealthCheck, - refreshETCDConfigToExist, - allETCDNodeHealthCheck, - } - return tasks -} - -func handleExistCluster(c *ConfigureModule) []task.Interface { - - existETCDHealthCheck := &task.RemoteTask{ - Name: "ExistETCDHealthCheck", - Desc: "Health check on exist etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: new(NodeETCDExist), - Action: new(HealthCheck), - Parallel: true, - Retry: 20, - } - - generateETCDConfig := &task.RemoteTask{ - Name: "GenerateETCDConfig", - Desc: "Generate etcd.env config on new etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: &NodeETCDExist{Not: true}, - Action: new(GenerateConfig), - Parallel: false, - } - - joinMember := &task.RemoteTask{ - Name: "JoinETCDMember", - Desc: "Join etcd member", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: &NodeETCDExist{Not: true}, - Action: new(JoinMember), - Parallel: false, - } - - newETCDNodeHealthCheck := &task.RemoteTask{ - Name: "NewETCDNodeHealthCheck", - Desc: "Health check on new etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: &NodeETCDExist{Not: true}, - Action: new(HealthCheck), - Parallel: true, - Retry: 20, - } - - checkMember := &task.RemoteTask{ - Name: "CheckETCDMember", - Desc: "Check etcd member", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Prepare: &NodeETCDExist{Not: true}, - Action: new(CheckMember), - Parallel: true, - } - - allRefreshETCDConfig := &task.RemoteTask{ - Name: "AllRefreshETCDConfig", - Desc: "Refresh etcd.env config on all etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Action: new(RefreshConfig), - Parallel: false, - } - - allETCDNodeHealthCheck := &task.RemoteTask{ - Name: "AllETCDNodeHealthCheck", - Desc: "Health check on all etcd", - Hosts: c.Runtime.GetHostsByRole(common.ETCD), - Action: new(HealthCheck), - Parallel: true, - Retry: 20, - } - - tasks := []task.Interface{ - existETCDHealthCheck, - generateETCDConfig, - joinMember, - newETCDNodeHealthCheck, - checkMember, - allRefreshETCDConfig, - allETCDNodeHealthCheck, - } - return tasks -} - -type BackupModule struct { - common.KubeModule - Skip bool -} - -func (p *BackupModule) IsSkip() bool { - return p.Skip -} - -func (b *BackupModule) Init() { - b.Name = "ETCDBackupModule" - b.Desc = "Backup ETCD cluster data" - - backupETCD := &task.RemoteTask{ - Name: "BackupETCD", - Desc: "Backup etcd data regularly", - Hosts: b.Runtime.GetHostsByRole(common.ETCD), - Action: new(BackupETCD), - Parallel: true, - } - - generateBackupETCDService := &task.RemoteTask{ - Name: "GenerateBackupETCDService", - Desc: "Generate backup ETCD service", - Hosts: b.Runtime.GetHostsByRole(common.ETCD), - Action: &action.Template{ - Template: templates.BackupETCDService, - Dst: filepath.Join("/etc/systemd/system/", templates.BackupETCDService.Name()), - Data: util.Data{ - "ScriptPath": filepath.Join(b.KubeConf.Cluster.Etcd.BackupScriptDir, "etcd-backup.sh"), - }, - }, - Parallel: true, - } - - generateBackupETCDTimer := &task.RemoteTask{ - Name: "GenerateBackupETCDTimer", - Desc: "Generate backup ETCD timer", - Hosts: b.Runtime.GetHostsByRole(common.ETCD), - Action: &action.Template{ - Template: templates.BackupETCDTimer, - Dst: filepath.Join("/etc/systemd/system/", templates.BackupETCDTimer.Name()), - Data: util.Data{ - "OnCalendarStr": templates.BackupTimeOnCalendar(b.KubeConf.Cluster.Etcd.BackupPeriod), - }, - }, - Parallel: true, - } - - enable := &task.RemoteTask{ - Name: "EnableBackupETCDService", - Desc: "Enable backup etcd service", - Hosts: b.Runtime.GetHostsByRole(common.ETCD), - Action: new(EnableBackupETCDService), - Parallel: true, - } - - b.Tasks = []task.Interface{ - backupETCD, - generateBackupETCDService, - generateBackupETCDTimer, - enable, - } -} diff --git a/cmd/kk/pkg/etcd/prepares.go b/cmd/kk/pkg/etcd/prepares.go deleted file mode 100644 index aa183dbd..00000000 --- a/cmd/kk/pkg/etcd/prepares.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package etcd - -import ( - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type FirstETCDNode struct { - common.KubePrepare - Not bool -} - -func (f *FirstETCDNode) PreCheck(runtime connector.Runtime) (bool, error) { - v, ok := f.PipelineCache.Get(common.ETCDCluster) - if !ok { - return false, errors.New("get etcd cluster status by pipeline cache failed") - } - cluster := v.(*EtcdCluster) - - if (!cluster.clusterExist && runtime.GetHostsByRole(common.ETCD)[0].GetName() == runtime.RemoteHost().GetName()) || - (cluster.clusterExist && strings.Contains(cluster.peerAddresses[0], runtime.RemoteHost().GetInternalAddress())) { - return !f.Not, nil - } - return f.Not, nil -} - -type NodeETCDExist struct { - common.KubePrepare - Not bool -} - -func (n *NodeETCDExist) PreCheck(runtime connector.Runtime) (bool, error) { - host := runtime.RemoteHost() - if v, ok := host.GetCache().GetMustBool(common.ETCDExist); ok { - if v { - return !n.Not, nil - } - return n.Not, nil - } - return false, errors.New("get etcd node status by host label failed") -} diff --git a/cmd/kk/pkg/etcd/tasks.go b/cmd/kk/pkg/etcd/tasks.go deleted file mode 100644 index 0aeaccdd..00000000 --- a/cmd/kk/pkg/etcd/tasks.go +++ /dev/null @@ -1,437 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package etcd - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/etcd/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -type EtcdNode struct { - NodeName string - EtcdName string - EtcdExist bool -} - -type EtcdCluster struct { - clusterExist bool - accessAddresses string - peerAddresses []string -} - -const ( - LocalCertsDir = "localCertsDir" - CertsFileList = "certsFileList" - - NewCluster = "new" - ExistCluster = "existing" -) - -type GetStatus struct { - common.KubeAction -} - -func (g *GetStatus) Execute(runtime connector.Runtime) error { - exist, err := runtime.GetRunner().FileExist("/etc/etcd.env") - if err != nil { - return err - } - - host := runtime.RemoteHost() - cluster := &EtcdCluster{ - clusterExist: true, - accessAddresses: "", - peerAddresses: []string{}, - } - - if exist { - etcdEnv, err := runtime.GetRunner().SudoCmd("cat /etc/etcd.env | grep ETCD_NAME", true) - if err != nil { - return err - } - - etcdName := etcdEnv[strings.Index(etcdEnv, "=")+1:] - // type: string - host.GetCache().Set(common.ETCDName, etcdName) - // type: bool - host.GetCache().Set(common.ETCDExist, true) - - if v, ok := g.PipelineCache.Get(common.ETCDCluster); ok { - c := v.(*EtcdCluster) - c.peerAddresses = append(c.peerAddresses, fmt.Sprintf("%s=https://%s:2380", etcdName, host.GetInternalAddress())) - c.clusterExist = true - // type: *EtcdCluster - g.PipelineCache.Set(common.ETCDCluster, c) - } else { - cluster.peerAddresses = append(cluster.peerAddresses, fmt.Sprintf("%s=https://%s:2380", etcdName, host.GetInternalAddress())) - cluster.clusterExist = true - g.PipelineCache.Set(common.ETCDCluster, cluster) - } - } else { - host.GetCache().Set(common.ETCDName, fmt.Sprintf("etcd-%s", host.GetName())) - host.GetCache().Set(common.ETCDExist, false) - - if _, ok := g.PipelineCache.Get(common.ETCDCluster); !ok { - cluster.clusterExist = false - g.PipelineCache.Set(common.ETCDCluster, cluster) - } - } - return nil -} - -type SyncCertsFile struct { - common.KubeAction -} - -func (s *SyncCertsFile) Execute(runtime connector.Runtime) error { - localCertsDir, ok := s.ModuleCache.Get(LocalCertsDir) - if !ok { - return errors.New("get etcd local certs dir by module cache failed") - } - files, ok := s.ModuleCache.Get(CertsFileList) - if !ok { - return errors.New("get etcd certs file list by module cache failed") - } - dir := localCertsDir.(string) - fileList := files.([]string) - - for _, fileName := range fileList { - if err := runtime.GetRunner().SudoScp(filepath.Join(dir, fileName), filepath.Join(common.ETCDCertDir, fileName)); err != nil { - return errors.Wrap(errors.WithStack(err), "scp etcd certs file failed") - } - } - - return nil -} - -type InstallETCDBinary struct { - common.KubeAction -} - -func (g *InstallETCDBinary) Execute(runtime connector.Runtime) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binariesMapObj, ok := g.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - binary, ok := binariesMap["etcd"] - if !ok { - return fmt.Errorf("get kube binary etcd info failed: no such key") - } - - dst := filepath.Join(common.TmpDir, binary.FileName) - if err := runtime.GetRunner().Scp(binary.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), "sync etcd tar.gz failed") - } - - etcdDir := strings.TrimSuffix(binary.FileName, ".tar.gz") - installCmd := fmt.Sprintf("tar -zxf %s && cp -f %s/etcd* /usr/local/bin/ && chmod +x /usr/local/bin/etcd* && rm -rf %s", dst, etcdDir, etcdDir) - if _, err := runtime.GetRunner().SudoCmd(installCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "install etcd binaries failed") - } - return nil -} - -type GenerateAccessAddress struct { - common.KubeAction -} - -func (g *GenerateAccessAddress) Execute(runtime connector.Runtime) error { - var addrList []string - for _, host := range runtime.GetHostsByRole(common.ETCD) { - addrList = append(addrList, fmt.Sprintf("https://%s:2379", host.GetInternalAddress())) - } - - accessAddresses := strings.Join(addrList, ",") - if v, ok := g.PipelineCache.Get(common.ETCDCluster); ok { - cluster := v.(*EtcdCluster) - cluster.accessAddresses = accessAddresses - g.PipelineCache.Set(common.ETCDCluster, cluster) - } else { - return errors.New("get etcd cluster status by pipeline cache failed") - } - return nil -} - -type HealthCheck struct { - common.KubeAction -} - -func (h *HealthCheck) Execute(runtime connector.Runtime) error { - if v, ok := h.PipelineCache.Get(common.ETCDCluster); ok { - cluster := v.(*EtcdCluster) - if err := healthCheck(runtime, cluster); err != nil { - return err - } - } else { - return errors.New("get etcd cluster status by pipeline cache failed") - } - return nil -} - -func healthCheck(runtime connector.Runtime, cluster *EtcdCluster) error { - host := runtime.RemoteHost() - checkHealthCmd := fmt.Sprintf("export ETCDCTL_API=2;"+ - "export ETCDCTL_CERT_FILE='/etc/ssl/etcd/ssl/admin-%s.pem';"+ - "export ETCDCTL_KEY_FILE='/etc/ssl/etcd/ssl/admin-%s-key.pem';"+ - "export ETCDCTL_CA_FILE='/etc/ssl/etcd/ssl/ca.pem';"+ - "%s/etcdctl --endpoints=%s cluster-health | grep -q 'cluster is healthy'", - host.GetName(), host.GetName(), common.BinDir, cluster.accessAddresses) - if _, err := runtime.GetRunner().SudoCmd(checkHealthCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "etcd health check failed") - } - return nil -} - -type GenerateConfig struct { - common.KubeAction -} - -func (g *GenerateConfig) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - etcdName, ok := host.GetCache().GetMustString(common.ETCDName) - if !ok { - return errors.New("get etcd node status by host label failed") - } - - if v, ok := g.PipelineCache.Get(common.ETCDCluster); ok { - cluster := v.(*EtcdCluster) - - cluster.peerAddresses = append(cluster.peerAddresses, fmt.Sprintf("%s=https://%s:2380", etcdName, host.GetInternalAddress())) - g.PipelineCache.Set(common.ETCDCluster, cluster) - - if !cluster.clusterExist { - if err := refreshConfig(g.KubeConf, runtime, cluster.peerAddresses, NewCluster, etcdName); err != nil { - return err - } - } else { - if err := refreshConfig(g.KubeConf, runtime, cluster.peerAddresses, ExistCluster, etcdName); err != nil { - return err - } - } - return nil - } else { - return errors.New("get etcd cluster status by pipeline cache failed") - } -} - -type RefreshConfig struct { - common.KubeAction - ToExisting bool -} - -func (r *RefreshConfig) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - etcdName, ok := host.GetCache().GetMustString(common.ETCDName) - if !ok { - return errors.New("get etcd node status by host label failed") - } - - if v, ok := r.PipelineCache.Get(common.ETCDCluster); ok { - cluster := v.(*EtcdCluster) - - if r.ToExisting { - if err := refreshConfig(r.KubeConf, runtime, cluster.peerAddresses, ExistCluster, etcdName); err != nil { - return err - } - return nil - } - - if !cluster.clusterExist { - if err := refreshConfig(r.KubeConf, runtime, cluster.peerAddresses, NewCluster, etcdName); err != nil { - return err - } - } else { - if err := refreshConfig(r.KubeConf, runtime, cluster.peerAddresses, ExistCluster, etcdName); err != nil { - return err - } - } - return nil - } - return errors.New("get etcd cluster status by pipeline cache failed") -} - -func refreshConfig(KubeConf *common.KubeConf, runtime connector.Runtime, endpoints []string, state, etcdName string) error { - host := runtime.RemoteHost() - - UnsupportedArch := false - if host.GetArch() != "amd64" { - UnsupportedArch = true - } - - templateAction := action.Template{ - Template: templates.EtcdEnv, - Dst: filepath.Join("/etc/", templates.EtcdEnv.Name()), - Data: util.Data{ - "Tag": kubekeyapiv1alpha2.DefaultEtcdVersion, - "Name": etcdName, - "Ip": host.GetInternalAddress(), - "Hostname": host.GetName(), - "State": state, - "PeerAddresses": strings.Join(endpoints, ","), - "UnsupportedArch": UnsupportedArch, - "Arch": host.GetArch(), - "DataDir": KubeConf.Cluster.Etcd.DataDir, - "CompactionRetention": KubeConf.Cluster.Etcd.AutoCompactionRetention, - "SnapshotCount": KubeConf.Cluster.Etcd.SnapshotCount, - "Metrics": KubeConf.Cluster.Etcd.Metrics, - "QuotaBackendBytes": KubeConf.Cluster.Etcd.QuotaBackendBytes, - "MaxRequestBytes": KubeConf.Cluster.Etcd.MaxRequestBytes, - "LogLevel": KubeConf.Cluster.Etcd.LogLevel, - "MaxSnapshots": KubeConf.Cluster.Etcd.MaxSnapshots, - "MaxWals": KubeConf.Cluster.Etcd.MaxWals, - "ElectionTimeout": KubeConf.Cluster.Etcd.ElectionTimeout, - "HeartbeatInterval": KubeConf.Cluster.Etcd.HeartbeatInterval, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type JoinMember struct { - common.KubeAction -} - -func (j *JoinMember) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - etcdName, ok := host.GetCache().GetMustString(common.ETCDName) - if !ok { - return errors.New("get etcd node status by host label failed") - } - - if v, ok := j.PipelineCache.Get(common.ETCDCluster); ok { - cluster := v.(*EtcdCluster) - joinMemberCmd := fmt.Sprintf("export ETCDCTL_API=2;"+ - "export ETCDCTL_CERT_FILE='/etc/ssl/etcd/ssl/admin-%s.pem';"+ - "export ETCDCTL_KEY_FILE='/etc/ssl/etcd/ssl/admin-%s-key.pem';"+ - "export ETCDCTL_CA_FILE='/etc/ssl/etcd/ssl/ca.pem';"+ - "%s/etcdctl --endpoints=%s member add %s %s", - host.GetName(), host.GetName(), common.BinDir, cluster.accessAddresses, etcdName, - fmt.Sprintf("https://%s:2380", host.GetInternalAddress())) - - if _, err := runtime.GetRunner().SudoCmd(joinMemberCmd, true); err != nil { - return errors.Wrap(errors.WithStack(err), "add etcd member failed") - } - } else { - return errors.New("get etcd cluster status by pipeline cache failed") - } - - // After adding a new member for etcd, it is necessary to start the new member as the etcd cluster may experience abnormal behavior. - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && systemctl restart etcd && systemctl enable etcd", true); err != nil { - return errors.Wrap(errors.WithStack(err), "start etcd failed") - } - - return nil -} - -type CheckMember struct { - common.KubeAction -} - -func (c *CheckMember) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - if v, ok := c.PipelineCache.Get(common.ETCDCluster); ok { - cluster := v.(*EtcdCluster) - checkMemberCmd := fmt.Sprintf("export ETCDCTL_API=2;"+ - "export ETCDCTL_CERT_FILE='/etc/ssl/etcd/ssl/admin-%s.pem';"+ - "export ETCDCTL_KEY_FILE='/etc/ssl/etcd/ssl/admin-%s-key.pem';"+ - "export ETCDCTL_CA_FILE='/etc/ssl/etcd/ssl/ca.pem';"+ - "%s/etcdctl --no-sync --endpoints=%s member list", host.GetName(), host.GetName(), common.BinDir, cluster.accessAddresses) - memberList, err := runtime.GetRunner().SudoCmd(checkMemberCmd, true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "list etcd member failed") - } - if !strings.Contains(memberList, fmt.Sprintf("https://%s:2379", host.GetInternalAddress())) { - return errors.Wrap(errors.WithStack(err), "add etcd member failed") - } - } else { - return errors.New("get etcd cluster status by pipeline cache failed") - } - return nil -} - -type RestartETCD struct { - common.KubeAction -} - -func (r *RestartETCD) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && systemctl restart etcd && systemctl enable etcd", true); err != nil { - return errors.Wrap(errors.WithStack(err), "start etcd failed") - } - return nil -} - -type BackupETCD struct { - common.KubeAction -} - -func (b *BackupETCD) Execute(runtime connector.Runtime) error { - templateAction := action.Template{ - Template: templates.EtcdBackupScript, - Dst: filepath.Join(b.KubeConf.Cluster.Etcd.BackupScriptDir, "etcd-backup.sh"), - Data: util.Data{ - "Hostname": runtime.RemoteHost().GetName(), - "Etcdendpoint": fmt.Sprintf("https://%s:2379", runtime.RemoteHost().GetInternalAddress()), - "DataDir": b.KubeConf.Cluster.Etcd.DataDir, - "Backupdir": b.KubeConf.Cluster.Etcd.BackupDir, - "KeepbackupNumber": b.KubeConf.Cluster.Etcd.KeepBackupNumber + 1, - "EtcdBackupScriptDir": b.KubeConf.Cluster.Etcd.BackupScriptDir, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s/etcd-backup.sh", b.KubeConf.Cluster.Etcd.BackupScriptDir), false); err != nil { - return errors.Wrap(errors.WithStack(err), "chmod etcd backup script failed") - } - return nil -} - -type EnableBackupETCDService struct { - common.KubeAction -} - -func (e *EnableBackupETCDService) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl enable --now backup-etcd.timer", - false); err != nil { - return errors.Wrap(errors.WithStack(err), "enable backup-etcd.service failed") - } - return nil -} diff --git a/cmd/kk/pkg/etcd/templates/backup_script.go b/cmd/kk/pkg/etcd/templates/backup_script.go deleted file mode 100644 index 9445aff1..00000000 --- a/cmd/kk/pkg/etcd/templates/backup_script.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -// EtcdBackupScriptTmpl defines the template of etcd backup script. -var EtcdBackupScript = template.Must(template.New("etcd-backup.sh").Parse( - dedent.Dedent(`#!/bin/bash - -set -o errexit -set -o nounset -set -o pipefail - -ETCDCTL_PATH='/usr/local/bin/etcdctl' -ENDPOINTS='{{ .Etcdendpoint }}' -{{- if .DataDir }} -ETCD_DATA_DIR="{{ .DataDir }}" -{{- else }} -ETCD_DATA_DIR="/var/lib/etcd" -{{- end }} -BACKUP_DIR="{{ .Backupdir }}/etcd-$(date +%Y-%m-%d-%H-%M-%S)" -KEEPBACKUPNUMBER='{{ .KeepbackupNumber }}' -ETCDBACKUPSCIPT='{{ .EtcdBackupScriptDir }}' - -ETCDCTL_CERT="/etc/ssl/etcd/ssl/admin-{{ .Hostname }}.pem" -ETCDCTL_KEY="/etc/ssl/etcd/ssl/admin-{{ .Hostname }}-key.pem" -ETCDCTL_CA_FILE="/etc/ssl/etcd/ssl/ca.pem" - -[ ! -d $BACKUP_DIR ] && mkdir -p $BACKUP_DIR - -export ETCDCTL_API=2;$ETCDCTL_PATH backup --data-dir $ETCD_DATA_DIR --backup-dir $BACKUP_DIR - -sleep 3 - -{ -export ETCDCTL_API=3;$ETCDCTL_PATH --endpoints="$ENDPOINTS" snapshot save $BACKUP_DIR/snapshot.db \ - --cacert="$ETCDCTL_CA_FILE" \ - --cert="$ETCDCTL_CERT" \ - --key="$ETCDCTL_KEY" -} > /dev/null - -sleep 3 - -cd $BACKUP_DIR/../ && ls -lt |awk '{if(NR > '$KEEPBACKUPNUMBER'){print "rm -rf "$9}}'|sh - -`))) diff --git a/cmd/kk/pkg/etcd/templates/backup_service.go b/cmd/kk/pkg/etcd/templates/backup_service.go deleted file mode 100644 index d2b25ddb..00000000 --- a/cmd/kk/pkg/etcd/templates/backup_service.go +++ /dev/null @@ -1,71 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "fmt" - "strconv" - "text/template" - - "github.com/lithammer/dedent" -) - -var ( - // BackupETCDService defines the template of backup-etcd service for systemd. - BackupETCDService = template.Must(template.New("backup-etcd.service").Parse( - dedent.Dedent(`[Unit] -Description=Backup ETCD -[Service] -Type=oneshot -ExecStart={{ .ScriptPath }} - `))) - - // BackupETCDTimer defines the template of backup-etcd timer for systemd. - BackupETCDTimer = template.Must(template.New("backup-etcd.timer").Parse( - dedent.Dedent(`[Unit] -Description=Timer to backup ETCD -[Timer] -{{- if .OnCalendarStr }} -OnCalendar={{ .OnCalendarStr }} -{{- else }} -OnCalendar=*-*-* 02:00:00 -{{- end }} -Unit=backup-etcd.service -[Install] -WantedBy=multi-user.target - `))) -) - -func BackupTimeOnCalendar(period int) string { - var onCalendar string - if period <= 0 { - onCalendar = "*-*-* *:00/30:00" - } else if period < 60 { - onCalendar = fmt.Sprintf("*-*-* *:00/%d:00", period) - } else if period >= 60 && period < 1440 { - hour := strconv.Itoa(period / 60) - minute := strconv.Itoa(period % 60) - if period%60 == 0 { - onCalendar = fmt.Sprintf("*-*-* 00/%s:00:00", hour) - } else { - onCalendar = fmt.Sprintf("*-*-* 00/%s:%s:00", hour, minute) - } - } else { - onCalendar = "*-*-* 02:00:00" - } - return onCalendar -} diff --git a/cmd/kk/pkg/etcd/templates/backup_service_test.go b/cmd/kk/pkg/etcd/templates/backup_service_test.go deleted file mode 100644 index 4abeba21..00000000 --- a/cmd/kk/pkg/etcd/templates/backup_service_test.go +++ /dev/null @@ -1,52 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "testing" -) - -func TestBackupTimeOnCalendar(t *testing.T) { - tests := []struct { - period int - want string - }{ - { - 30, - "*-*-* *:00/30:00", - }, - { - 60, - "*-*-* 00/1:00:00", - }, - { - 70, - "*-*-* 00/1:10:00", - }, - { - 1500, - "*-*-* 00:00:00", - }, - } - for _, tt := range tests { - t.Run("", func(t *testing.T) { - if got := BackupTimeOnCalendar(tt.period); got != tt.want { - t.Errorf("BackupTimeOnCalendar() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/cmd/kk/pkg/etcd/templates/etcd_env.go b/cmd/kk/pkg/etcd/templates/etcd_env.go deleted file mode 100644 index f6506c76..00000000 --- a/cmd/kk/pkg/etcd/templates/etcd_env.go +++ /dev/null @@ -1,101 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -// EtcdEnv defines the template of etcd's env. -var EtcdEnv = template.Must(template.New("etcd.env").Parse( - dedent.Dedent(`# Environment file for etcd {{ .Tag }} -{{- if .DataDir }} -ETCD_DATA_DIR={{ .DataDir }} -{{- else }} -ETCD_DATA_DIR=/var/lib/etcd -{{- end }} -ETCD_ADVERTISE_CLIENT_URLS=https://{{ .Ip }}:2379 -ETCD_INITIAL_ADVERTISE_PEER_URLS=https://{{ .Ip }}:2380 -ETCD_INITIAL_CLUSTER_STATE={{ .State }} -ETCD_METRICS=basic -ETCD_LISTEN_CLIENT_URLS=https://{{ .Ip }}:2379,https://127.0.0.1:2379 -ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd -ETCD_LISTEN_PEER_URLS=https://{{ .Ip }}:2380 -ETCD_NAME={{ .Name }} -ETCD_PROXY=off -ETCD_ENABLE_V2=true -ETCD_INITIAL_CLUSTER={{ .PeerAddresses }} -{{- if .ElectionTimeout }} -ETCD_ELECTION_TIMEOUT={{ .ElectionTimeout }} -{{- else }} -ETCD_ELECTION_TIMEOUT=5000 -{{- end }} -{{- if .HeartbeatInterval }} -ETCD_HEARTBEAT_INTERVAL={{ .HeartbeatInterval }} -{{- else }} -ETCD_HEARTBEAT_INTERVAL=250 -{{- end }} -{{- if .CompactionRetention }} -ETCD_AUTO_COMPACTION_RETENTION={{ .CompactionRetention }} -{{- else }} -ETCD_AUTO_COMPACTION_RETENTION=8 -{{- end }} -{{- if .SnapshotCount }} -ETCD_SNAPSHOT_COUNT={{ .SnapshotCount }} -{{- else }} -ETCD_SNAPSHOT_COUNT=10000 -{{- end }} -{{- if .Metrics }} -ETCD_METRICS={{ .Metrics }} -{{- end }} -{{- if .QuotaBackendBytes }} -ETCD_QUOTA_BACKEND_BYTES={{ .QuotaBackendBytes }} -{{- end }} -{{- if .MaxRequestBytes }} -ETCD_MAX_REQUEST_BYTES={{ .MaxRequestBytes }} -{{- end }} -{{- if .MaxSnapshots }} -ETCD_MAX_SNAPSHOTS={{ .MaxSnapshots }} -{{- end }} -{{- if .MaxWals }} -ETCD_MAX_WALS={{ .MaxWals }} -{{- end }} -{{- if .LogLevel }} -ETCD_LOG_LEVEL={{ .LogLevel }} -{{- end }} -{{- if .UnsupportedArch }} -ETCD_UNSUPPORTED_ARCH={{ .Arch }} -{{ end }} - -# TLS settings -ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem -ETCD_CERT_FILE=/etc/ssl/etcd/ssl/member-{{ .Hostname }}.pem -ETCD_KEY_FILE=/etc/ssl/etcd/ssl/member-{{ .Hostname }}-key.pem -ETCD_CLIENT_CERT_AUTH=true - -ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem -ETCD_PEER_CERT_FILE=/etc/ssl/etcd/ssl/member-{{ .Hostname }}.pem -ETCD_PEER_KEY_FILE=/etc/ssl/etcd/ssl/member-{{ .Hostname }}-key.pem -ETCD_PEER_CLIENT_CERT_AUTH=True - -# CLI settings -ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 -ETCDCTL_CACERT=/etc/ssl/etcd/ssl/ca.pem -ETCDCTL_KEY=/etc/ssl/etcd/ssl/admin-{{ .Hostname }}-key.pem -ETCDCTL_CERT=/etc/ssl/etcd/ssl/admin-{{ .Hostname }}.pem - `))) diff --git a/cmd/kk/pkg/etcd/templates/etcd_service.go b/cmd/kk/pkg/etcd/templates/etcd_service.go deleted file mode 100644 index 78f2b788..00000000 --- a/cmd/kk/pkg/etcd/templates/etcd_service.go +++ /dev/null @@ -1,44 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var ( - // ETCDService defines the template of etcd's service for systemd. - ETCDService = template.Must(template.New("etcd.service").Parse( - dedent.Dedent(`[Unit] -Description=etcd -After=network.target - -[Service] -User=root -Type=notify -EnvironmentFile=/etc/etcd.env -ExecStart=/usr/local/bin/etcd -NotifyAccess=all -RestartSec=10s -LimitNOFILE=40000 -Restart=always - -[Install] -WantedBy=multi-user.target - `))) -) diff --git a/cmd/kk/pkg/files/file.go b/cmd/kk/pkg/files/file.go deleted file mode 100644 index bba319b6..00000000 --- a/cmd/kk/pkg/files/file.go +++ /dev/null @@ -1,328 +0,0 @@ -/* - Copyright 2020 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package files - -import ( - "crypto/sha256" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/version" -) - -const ( - kubeadm = "kubeadm" - kubelet = "kubelet" - kubectl = "kubectl" - kubecni = "kubecni" - etcd = "etcd" - helm = "helm" - amd64 = "amd64" - arm64 = "arm64" - k3s = "k3s" - k8e = "k8e" - docker = "docker" - crictl = "crictl" - registry = "registry" - harbor = "harbor" - compose = "compose" - containerd = "containerd" - runc = "runc" - calicoctl = "calicoctl" -) - -// KubeBinary Type field const -const ( - CNI = "cni" - CRICTL = "crictl" - DOCKER = "docker" - ETCD = "etcd" - HELM = "helm" - KUBE = "kube" - REGISTRY = "registry" - CONTAINERD = "containerd" - RUNC = "runc" -) - -var ( - // FileSha256 is a hash table the storage the checksum of the binary files. It is parsed from 'version/components.json'. - FileSha256 = map[string]map[string]map[string]string{} -) - -func init() { - FileSha256, _ = version.ParseFilesSha256(version.Components) -} - -type KubeBinary struct { - Type string - ID string - FileName string - Arch string - Version string - Url string - BaseDir string - Zone string - getCmd func(path, url string) string -} - -func NewKubeBinary(name, arch, version, prePath string, getCmd func(path, url string) string) *KubeBinary { - component := new(KubeBinary) - component.ID = name - component.Arch = arch - component.Version = version - component.Zone = os.Getenv("KKZONE") - component.getCmd = getCmd - - switch name { - case etcd: - component.Type = ETCD - component.FileName = fmt.Sprintf("etcd-%s-linux-%s.tar.gz", version, arch) - component.Url = fmt.Sprintf("https://github.com/coreos/etcd/releases/download/%s/etcd-%s-linux-%s.tar.gz", version, version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf( - "https://kubernetes-release.pek3b.qingstor.com/etcd/release/download/%s/etcd-%s-linux-%s.tar.gz", - component.Version, component.Version, component.Arch) - } - case kubeadm: - component.Type = KUBE - component.FileName = kubeadm - component.Url = fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/release/%s/bin/linux/%s/kubeadm", version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/release/%s/bin/linux/%s/kubeadm", version, arch) - } - case kubelet: - component.Type = KUBE - component.FileName = kubelet - component.Url = fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/release/%s/bin/linux/%s/kubelet", version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/release/%s/bin/linux/%s/kubelet", version, arch) - } - case kubectl: - component.Type = KUBE - component.FileName = kubectl - component.Url = fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/release/%s/bin/linux/%s/kubectl", version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/release/%s/bin/linux/%s/kubectl", version, arch) - } - case kubecni: - component.Type = CNI - component.FileName = fmt.Sprintf("cni-plugins-linux-%s-%s.tgz", arch, version) - component.Url = fmt.Sprintf("https://github.com/containernetworking/plugins/releases/download/%s/cni-plugins-linux-%s-%s.tgz", version, arch, version) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://containernetworking.pek3b.qingstor.com/plugins/releases/download/%s/cni-plugins-linux-%s-%s.tgz", version, arch, version) - } - case helm: - component.Type = HELM - component.FileName = helm - component.Url = fmt.Sprintf("https://get.helm.sh/helm-%s-linux-%s.tar.gz", version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-helm.pek3b.qingstor.com/linux-%s/%s/helm", arch, version) - } - case docker: - component.Type = DOCKER - component.FileName = fmt.Sprintf("docker-%s.tgz", version) - component.Url = fmt.Sprintf("https://download.docker.com/linux/static/stable/%s/docker-%s.tgz", util.ArchAlias(arch), version) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://mirrors.aliyun.com/docker-ce/linux/static/stable/%s/docker-%s.tgz", util.ArchAlias(arch), version) - } - case crictl: - component.Type = CRICTL - component.FileName = fmt.Sprintf("crictl-%s-linux-%s.tar.gz", version, arch) - component.Url = fmt.Sprintf("https://github.com/kubernetes-sigs/cri-tools/releases/download/%s/crictl-%s-linux-%s.tar.gz", version, version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/cri-tools/releases/download/%s/crictl-%s-linux-%s.tar.gz", version, version, arch) - } - case k3s: - component.Type = KUBE - component.FileName = k3s - component.Url = fmt.Sprintf("https://github.com/k3s-io/k3s/releases/download/%s+k3s1/k3s", version) - if arch == arm64 { - component.Url = fmt.Sprintf("https://github.com/k3s-io/k3s/releases/download/%s+k3s1/k3s-%s", version, arch) - } - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/k3s/releases/download/%s+k3s1/linux/%s/k3s", version, arch) - } - case k8e: - component.Type = KUBE - component.FileName = k8e - component.Url = fmt.Sprintf("https://github.com/xiaods/k8e/releases/download/%s+k8e2/k8e", version) - if arch == arm64 { - component.Url = fmt.Sprintf("https://github.com/xiaods/k8e/releases/download/%s+k8e2/k8e-%s", version, arch) - } - case registry: - component.Type = REGISTRY - component.FileName = fmt.Sprintf("registry-%s-linux-%s.tar.gz", version, arch) - component.Url = fmt.Sprintf("https://github.com/kubesphere/kubekey/releases/download/v2.0.0-alpha.1/registry-%s-linux-%s.tar.gz", version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/registry/%s/registry-%s-linux-%s.tar.gz", version, version, arch) - } - component.BaseDir = filepath.Join(prePath, component.Type, component.ID, component.Version, component.Arch) - case harbor: - component.Type = REGISTRY - component.FileName = fmt.Sprintf("harbor-offline-installer-%s.tgz", version) - component.Url = fmt.Sprintf("https://github.com/goharbor/harbor/releases/download/%s/harbor-offline-installer-%s.tgz", version, version) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/harbor/releases/download/%s/harbor-offline-installer-%s.tgz", version, version) - } - component.BaseDir = filepath.Join(prePath, component.Type, component.ID, component.Version, component.Arch) - case compose: - component.Type = REGISTRY - component.FileName = "docker-compose-linux-x86_64" - component.Url = fmt.Sprintf("https://github.com/docker/compose/releases/download/%s/docker-compose-linux-%s", version, util.ArchAlias(arch)) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/docker/compose/releases/download/%s/docker-compose-linux-%s", version, util.ArchAlias(arch)) - } - component.BaseDir = filepath.Join(prePath, component.Type, component.ID, component.Version, component.Arch) - case containerd: - component.Type = CONTAINERD - component.FileName = fmt.Sprintf("containerd-%s-linux-%s.tar.gz", version, arch) - component.Url = fmt.Sprintf("https://github.com/containerd/containerd/releases/download/v%s/containerd-%s-linux-%s.tar.gz", version, version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/containerd/containerd/releases/download/v%s/containerd-%s-linux-%s.tar.gz", version, version, arch) - } - case runc: - component.Type = RUNC - component.FileName = fmt.Sprintf("runc.%s", arch) - component.Url = fmt.Sprintf("https://github.com/opencontainers/runc/releases/download/%s/runc.%s", version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/opencontainers/runc/releases/download/%s/runc.%s", version, arch) - } - case calicoctl: - component.Type = CNI - component.FileName = calicoctl - component.Url = fmt.Sprintf("https://github.com/projectcalico/calico/releases/download/%s/calicoctl-linux-%s", version, arch) - if component.Zone == "cn" { - component.Url = fmt.Sprintf("https://kubernetes-release.pek3b.qingstor.com/projectcalico/calico/releases/download/%s/calicoctl-linux-%s", version, arch) - } - default: - logger.Log.Fatalf("unsupported kube binaries %s", name) - } - - if component.BaseDir == "" { - component.BaseDir = filepath.Join(prePath, component.Type, component.Version, component.Arch) - } - - return component -} - -func (b *KubeBinary) CreateBaseDir() error { - if err := util.CreateDir(b.BaseDir); err != nil { - return err - } - return nil -} - -func (b *KubeBinary) Path() string { - return filepath.Join(b.BaseDir, b.FileName) -} - -func (b *KubeBinary) GetCmd() string { - cmd := b.getCmd(b.Path(), b.Url) - - if b.ID == helm && b.Zone != "cn" { - get := b.getCmd(filepath.Join(b.BaseDir, fmt.Sprintf("helm-%s-linux-%s.tar.gz", b.Version, b.Arch)), b.Url) - cmd = fmt.Sprintf("%s && cd %s && tar -zxf helm-%s-linux-%s.tar.gz && mv linux-%s/helm . && rm -rf *linux-%s*", - get, b.BaseDir, b.Version, b.Arch, b.Arch, b.Arch) - } - return cmd -} - -func (b *KubeBinary) GetSha256() string { - s := FileSha256[b.ID][b.Arch][b.Version] - return s -} - -func (b *KubeBinary) Download() error { - for i := 5; i > 0; i-- { - cmd := exec.Command("/bin/sh", "-c", b.GetCmd()) - stdout, err := cmd.StdoutPipe() - if err != nil { - return err - } - cmd.Stderr = cmd.Stdout - - if err = cmd.Start(); err != nil { - return err - } - for { - tmp := make([]byte, 1024) - _, err := stdout.Read(tmp) - fmt.Print(string(tmp)) // Get the output from the pipeline in real time and print it to the terminal - if errors.Is(err, io.EOF) { - break - } else if err != nil { - logger.Log.Errorln(err) - break - } - } - if err = cmd.Wait(); err != nil { - if os.Getenv("KKZONE") != "cn" { - logger.Log.Warningln("Having a problem with accessing https://storage.googleapis.com? You can try again after setting environment 'export KKZONE=cn'") - } - return err - } - - if err := b.SHA256Check(); err != nil { - if i == 1 { - return err - } - path := b.Path() - _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", path)).Run() - continue - } - break - } - return nil -} - -// SHA256Check is used to hash checks on downloaded binary. (sha256) -func (b *KubeBinary) SHA256Check() error { - output, err := sha256sum(b.Path()) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to check SHA256 of %s", b.Path())) - } - - if strings.TrimSpace(b.GetSha256()) == "" { - return errors.New(fmt.Sprintf("No SHA256 found for %s. %s is not supported.", b.ID, b.Version)) - } - if output != b.GetSha256() { - return errors.New(fmt.Sprintf("SHA256 no match. %s not equal %s", b.GetSha256(), output)) - } - return nil -} - -func sha256sum(path string) (string, error) { - file, err := os.Open(path) - if err != nil { - return "", err - } - defer file.Close() - - hasher := sha256.New() - if _, err := io.Copy(hasher, file); err != nil { - return "", err - } - return fmt.Sprintf("%x", hasher.Sum(nil)), nil -} diff --git a/cmd/kk/pkg/filesystem/module.go b/cmd/kk/pkg/filesystem/module.go deleted file mode 100644 index fa0dacbe..00000000 --- a/cmd/kk/pkg/filesystem/module.go +++ /dev/null @@ -1,82 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package filesystem - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type ChownModule struct { - module.BaseTaskModule -} - -func (c *ChownModule) Init() { - c.Name = "ChownModule" - c.Desc = "Change file and dir mode and owner" - - userKubeDir := &task.RemoteTask{ - Name: "ChownFileAndDir", - Desc: "Chown user $HOME/.kube dir", - Hosts: c.Runtime.GetHostsByRole(common.K8s), - Action: &ChownFileAndDir{Path: "$HOME/.kube"}, - Parallel: true, - } - - c.Tasks = []task.Interface{ - userKubeDir, - } -} - -type ChownWorkDirModule struct { - module.BaseTaskModule -} - -func (c *ChownWorkDirModule) Init() { - c.Name = "ChownWorkerModule" - c.Desc = "Change kubekey work dir mode and owner" - - userKubeDir := &task.LocalTask{ - Name: "ChownFileAndDir", - Desc: "Chown ./kubekey dir", - Action: &LocalTaskChown{Path: c.Runtime.GetWorkDir()}, - } - - c.Tasks = []task.Interface{ - userKubeDir, - } -} - -type ChownOutputModule struct { - common.ArtifactModule -} - -func (c *ChownOutputModule) Init() { - c.Name = "ChownOutputModule" - c.Desc = "Change file and dir owner" - - output := &task.LocalTask{ - Name: "Chown output file", - Desc: "Chown output file", - Action: &LocalTaskChown{Path: c.Manifest.Arg.Output}, - } - - c.Tasks = []task.Interface{ - output, - } -} diff --git a/cmd/kk/pkg/filesystem/task.go b/cmd/kk/pkg/filesystem/task.go deleted file mode 100644 index b824e782..00000000 --- a/cmd/kk/pkg/filesystem/task.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package filesystem - -import ( - "fmt" - "os/exec" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" -) - -type ChownFileAndDir struct { - action.BaseAction - Path string -} - -func (c *ChownFileAndDir) Execute(runtime connector.Runtime) error { - exist, err := runtime.GetRunner().FileExist(c.Path) - if err != nil { - return errors.Wrapf(errors.WithStack(err), "get user %s failed", c.Path) - } - - if exist { - userId, err := runtime.GetRunner().Cmd("echo $(id -u)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user id failed") - } - - userGroupId, err := runtime.GetRunner().Cmd("echo $(id -g)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user group id failed") - } - - chownKubeConfig := fmt.Sprintf("chown -R %s:%s %s", userId, userGroupId, c.Path) - if _, err := runtime.GetRunner().SudoCmd(chownKubeConfig, false); err != nil { - return errors.Wrapf(errors.WithStack(err), "chown user %s failed", c.Path) - } - } - return nil -} - -type LocalTaskChown struct { - action.BaseAction - Path string -} - -func (l *LocalTaskChown) Execute(runtime connector.Runtime) error { - if exist := util.IsExist(l.Path); exist { - if err := exec.Command("/bin/sh", "-c", fmt.Sprintf("chown -R ${SUDO_UID}:${SUDO_GID} %s", l.Path)).Run(); err != nil { - return errors.Wrapf(errors.WithStack(err), "chown %s failed", l.Path) - } - } - return nil -} diff --git a/cmd/kk/pkg/images/copy.go b/cmd/kk/pkg/images/copy.go deleted file mode 100644 index 4d4cb2e5..00000000 --- a/cmd/kk/pkg/images/copy.go +++ /dev/null @@ -1,86 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "context" - "os" - - "github.com/containers/image/v5/copy" - "github.com/containers/image/v5/signature" - "github.com/containers/image/v5/transports/alltransports" -) - -type CopyImageOptions struct { - srcImage *srcImageOptions - destImage *destImageOptions - imageListSelection copy.ImageListSelection -} - -func (c *CopyImageOptions) Copy() error { - policyContext, err := getPolicyContext() - if err != nil { - return err - } - defer policyContext.Destroy() - - srcRef, err := alltransports.ParseImageName(c.srcImage.imageName) - if err != nil { - return err - } - destRef, err := alltransports.ParseImageName(c.destImage.imageName) - if err != nil { - return err - } - - srcContext := c.srcImage.systemContext() - destContext := c.destImage.systemContext() - - _, err = copy.Image(context.Background(), policyContext, destRef, srcRef, ©.Options{ - ReportWriter: os.Stdout, - SourceCtx: srcContext, - DestinationCtx: destContext, - ImageListSelection: c.imageListSelection, - }) - if err != nil { - return err - } - return nil -} - -func getPolicyContext() (*signature.PolicyContext, error) { - policy := &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}} - return signature.NewPolicyContext(policy) -} - -type Index struct { - Manifests []Manifest -} - -type Manifest struct { - Annotations annotations -} - -type annotations struct { - RefName string `json:"org.opencontainers.image.ref.name"` -} - -func NewIndex() *Index { - return &Index{ - Manifests: []Manifest{}, - } -} diff --git a/cmd/kk/pkg/images/images.go b/cmd/kk/pkg/images/images.go deleted file mode 100644 index f314cbea..00000000 --- a/cmd/kk/pkg/images/images.go +++ /dev/null @@ -1,151 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "fmt" - "os" - - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -const ( - cnRegistry = "registry.cn-beijing.aliyuncs.com" - cnNamespaceOverride = "kubesphereio" -) - -// Image defines image's info. -type Image struct { - RepoAddr string - Namespace string - NamespaceOverride string - Repo string - Tag string - Group string - Enable bool -} - -// Images contains a list of Image -type Images struct { - Images []Image -} - -// ImageName is used to generate image's full name. -func (image Image) ImageName() string { - return fmt.Sprintf("%s:%s", image.ImageRepo(), image.Tag) -} - -// ImageNamespace is used to get image's namespace -func (image Image) ImageNamespace() string { - if os.Getenv("KKZONE") == "cn" { - if image.RepoAddr == "" || image.RepoAddr == cnRegistry { - image.NamespaceOverride = cnNamespaceOverride - } - } - - if image.NamespaceOverride != "" { - return image.NamespaceOverride - } else { - return image.Namespace - } -} - -// ImageRegistryAddr is used to get image's registry address. -func (image Image) ImageRegistryAddr() string { - if os.Getenv("KKZONE") == "cn" { - if image.RepoAddr == "" || image.RepoAddr == cnRegistry { - image.RepoAddr = cnRegistry - } - } - if image.RepoAddr != "" { - return image.RepoAddr - } else { - return "docker.io" - } -} - -// ImageRepo is used to generate image's repo address. -func (image Image) ImageRepo() string { - var prefix string - - if os.Getenv("KKZONE") == "cn" { - if image.RepoAddr == "" || image.RepoAddr == cnRegistry { - image.RepoAddr = cnRegistry - image.NamespaceOverride = cnNamespaceOverride - } - } - - if image.RepoAddr == "" { - if image.Namespace == "" { - prefix = "" - } else { - prefix = fmt.Sprintf("%s/", image.Namespace) - } - } else { - if image.NamespaceOverride == "" { - if image.Namespace == "" { - prefix = fmt.Sprintf("%s/library/", image.RepoAddr) - } else { - prefix = fmt.Sprintf("%s/%s/", image.RepoAddr, image.Namespace) - } - } else { - prefix = fmt.Sprintf("%s/%s/", image.RepoAddr, image.NamespaceOverride) - } - } - - return fmt.Sprintf("%s%s", prefix, image.Repo) -} - -// PullImages is used to pull images in the list of Image. -func (images *Images) PullImages(runtime connector.Runtime, kubeConf *common.KubeConf) error { - pullCmd := "docker" - switch kubeConf.Cluster.Kubernetes.ContainerManager { - case "crio": - pullCmd = "crictl" - case "containerd": - pullCmd = "crictl" - case "isula": - pullCmd = "isula" - default: - pullCmd = "docker" - } - - host := runtime.RemoteHost() - - for _, image := range images.Images { - switch { - case host.IsRole(common.Master) && image.Group == kubekeyapiv1alpha2.Master && image.Enable, - host.IsRole(common.Worker) && image.Group == kubekeyapiv1alpha2.Worker && image.Enable, - (host.IsRole(common.Master) || host.IsRole(common.Worker)) && image.Group == kubekeyapiv1alpha2.K8s && image.Enable, - host.IsRole(common.ETCD) && image.Group == kubekeyapiv1alpha2.Etcd && image.Enable: - - logger.Log.Messagef(host.GetName(), "downloading image: %s", image.ImageName()) - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("env PATH=$PATH %s pull %s --platform %s", pullCmd, image.ImageName(), host.GetArch()), false); err != nil { - return errors.Wrap(err, "pull image failed") - } - default: - continue - } - - } - return nil -} diff --git a/cmd/kk/pkg/images/module.go b/cmd/kk/pkg/images/module.go deleted file mode 100644 index cc4843e0..00000000 --- a/cmd/kk/pkg/images/module.go +++ /dev/null @@ -1,99 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type PullModule struct { - common.KubeModule - Skip bool -} - -func (p *PullModule) IsSkip() bool { - return p.Skip -} - -func (p *PullModule) Init() { - p.Name = "PullModule" - p.Desc = "Pull images on all nodes" - - pull := &task.RemoteTask{ - Name: "PullImages", - Desc: "Start to pull images on all nodes", - Hosts: p.Runtime.GetAllHosts(), - Action: new(PullImage), - Parallel: true, - } - - p.Tasks = []task.Interface{ - pull, - } -} - -type CopyImagesToLocalModule struct { - common.ArtifactModule -} - -func (c *CopyImagesToLocalModule) Init() { - c.Name = "CopyImagesToLocalModule" - c.Desc = "Copy images to a local OCI path from registries" - - copyImage := &task.LocalTask{ - Name: "SaveImages", - Desc: "Copy images to a local OCI path from registries", - Action: new(SaveImages), - } - - c.Tasks = []task.Interface{ - copyImage, - } -} - -type CopyImagesToRegistryModule struct { - common.KubeModule - Skip bool - ImagePath string -} - -func (c *CopyImagesToRegistryModule) IsSkip() bool { - return c.Skip -} - -func (c *CopyImagesToRegistryModule) Init() { - c.Name = "CopyImagesToRegistryModule" - c.Desc = "Copy images to a private registry from an artifact OCI path" - - copyImage := &task.LocalTask{ - Name: "CopyImagesToRegistry", - Desc: "Copy images to a private registry from an artifact OCI Path", - Action: &CopyImagesToRegistry{ImagesPath: c.ImagePath}, - } - - pushManifest := &task.LocalTask{ - Name: "PushManifest", - Desc: "Push multi-arch manifest to private registry", - Action: new(PushManifest), - } - - c.Tasks = []task.Interface{ - copyImage, - pushManifest, - } -} diff --git a/cmd/kk/pkg/images/tasks.go b/cmd/kk/pkg/images/tasks.go deleted file mode 100644 index f8d59f8d..00000000 --- a/cmd/kk/pkg/images/tasks.go +++ /dev/null @@ -1,371 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "reflect" - "strings" - - manifestregistry "github.com/estesp/manifest-tool/v2/pkg/registry" - manifesttypes "github.com/estesp/manifest-tool/v2/pkg/types" - "github.com/pkg/errors" - versionutil "k8s.io/apimachinery/pkg/util/version" - - kubekeyv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - coreutil "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry" -) - -type PullImage struct { - common.KubeAction -} - -func (p *PullImage) Execute(runtime connector.Runtime) error { - if !p.KubeConf.Arg.SkipPullImages { - i := Images{} - i.Images = []Image{ - GetImage(runtime, p.KubeConf, "etcd"), - GetImage(runtime, p.KubeConf, "pause"), - GetImage(runtime, p.KubeConf, "kube-apiserver"), - GetImage(runtime, p.KubeConf, "kube-controller-manager"), - GetImage(runtime, p.KubeConf, "kube-scheduler"), - GetImage(runtime, p.KubeConf, "kube-proxy"), - GetImage(runtime, p.KubeConf, "coredns"), - GetImage(runtime, p.KubeConf, "k8s-dns-node-cache"), - GetImage(runtime, p.KubeConf, "calico-kube-controllers"), - GetImage(runtime, p.KubeConf, "calico-cni"), - GetImage(runtime, p.KubeConf, "calico-node"), - GetImage(runtime, p.KubeConf, "calico-flexvol"), - GetImage(runtime, p.KubeConf, "cilium"), - GetImage(runtime, p.KubeConf, "cilium-operator-generic"), - GetImage(runtime, p.KubeConf, "flannel"), - GetImage(runtime, p.KubeConf, "flannel-cni-plugin"), - GetImage(runtime, p.KubeConf, "kubeovn"), - GetImage(runtime, p.KubeConf, "haproxy"), - GetImage(runtime, p.KubeConf, "kubevip"), - } - - if err := i.PullImages(runtime, p.KubeConf); err != nil { - return err - } - } - return nil -} - -// GetImage defines the list of all images and gets image object by name. -func GetImage(runtime connector.ModuleRuntime, kubeConf *common.KubeConf, name string) Image { - var image Image - pauseTag, corednsTag := "3.2", "1.6.9" - - if versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).LessThan(versionutil.MustParseSemantic("v1.21.0")) { - pauseTag = "3.2" - corednsTag = "1.6.9" - } - if versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.21.0")) || - (kubeConf.Cluster.Kubernetes.ContainerManager != "" && kubeConf.Cluster.Kubernetes.ContainerManager != "docker") { - pauseTag = "3.4.1" - corednsTag = "1.8.0" - } - if versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.22.0")) { - pauseTag = "3.5" - corednsTag = "1.8.0" - } - if versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.23.0")) { - pauseTag = "3.6" - corednsTag = "1.8.6" - } - if versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.24.0")) { - pauseTag = "3.7" - corednsTag = "1.8.6" - } - if versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.25.0")) { - pauseTag = "3.8" - corednsTag = "1.9.3" - } - - logger.Log.Debugf("pauseTag: %s, corednsTag: %s", pauseTag, corednsTag) - - ImageList := map[string]Image{ - "pause": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "pause", Tag: pauseTag, Group: kubekeyv1alpha2.K8s, Enable: true}, - "etcd": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "etcd", Tag: kubekeyv1alpha2.DefaultEtcdVersion, Group: kubekeyv1alpha2.Master, Enable: strings.EqualFold(kubeConf.Cluster.Etcd.Type, kubekeyv1alpha2.Kubeadm)}, - "kube-apiserver": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "kube-apiserver", Tag: kubeConf.Cluster.Kubernetes.Version, Group: kubekeyv1alpha2.Master, Enable: true}, - "kube-controller-manager": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "kube-controller-manager", Tag: kubeConf.Cluster.Kubernetes.Version, Group: kubekeyv1alpha2.Master, Enable: true}, - "kube-scheduler": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "kube-scheduler", Tag: kubeConf.Cluster.Kubernetes.Version, Group: kubekeyv1alpha2.Master, Enable: true}, - "kube-proxy": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "kube-proxy", Tag: kubeConf.Cluster.Kubernetes.Version, Group: kubekeyv1alpha2.K8s, Enable: !kubeConf.Cluster.Kubernetes.DisableKubeProxy}, - - // network - "coredns": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "coredns", Repo: "coredns", Tag: corednsTag, Group: kubekeyv1alpha2.K8s, Enable: true}, - "k8s-dns-node-cache": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "k8s-dns-node-cache", Tag: "1.22.20", Group: kubekeyv1alpha2.K8s, Enable: kubeConf.Cluster.Kubernetes.EnableNodelocaldns()}, - "calico-kube-controllers": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "calico", Repo: "kube-controllers", Tag: kubekeyv1alpha2.DefaultCalicoVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "calico")}, - "calico-cni": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "calico", Repo: "cni", Tag: kubekeyv1alpha2.DefaultCalicoVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "calico")}, - "calico-node": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "calico", Repo: "node", Tag: kubekeyv1alpha2.DefaultCalicoVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "calico")}, - "calico-flexvol": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "calico", Repo: "pod2daemon-flexvol", Tag: kubekeyv1alpha2.DefaultCalicoVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "calico")}, - "calico-typha": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "calico", Repo: "typha", Tag: kubekeyv1alpha2.DefaultCalicoVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "calico") && len(runtime.GetHostsByRole(common.K8s)) > 50}, - "flannel": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "flannel", Repo: "flannel", Tag: kubekeyv1alpha2.DefaultFlannelVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "flannel")}, - "flannel-cni-plugin": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "flannel", Repo: "flannel-cni-plugin", Tag: kubekeyv1alpha2.DefaultFlannelCniPluginVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "flannel")}, - "cilium": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "cilium", Repo: "cilium", Tag: kubekeyv1alpha2.DefaultCiliumVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "cilium")}, - "cilium-operator-generic": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "cilium", Repo: "operator-generic", Tag: kubekeyv1alpha2.DefaultCiliumVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "cilium")}, - "hybridnet": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "hybridnetdev", Repo: "hybridnet", Tag: kubekeyv1alpha2.DefaulthybridnetVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "hybridnet")}, - "kubeovn": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "kubeovn", Repo: "kube-ovn", Tag: kubekeyv1alpha2.DefaultKubeovnVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "kubeovn")}, - "multus": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "multus-cni", Tag: kubekeyv1alpha2.DefalutMultusVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.Contains(kubeConf.Cluster.Network.Plugin, "multus")}, - // storage - "provisioner-localpv": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "openebs", Repo: "provisioner-localpv", Tag: "3.3.0", Group: kubekeyv1alpha2.Worker, Enable: false}, - "linux-utils": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "openebs", Repo: "linux-utils", Tag: "3.3.0", Group: kubekeyv1alpha2.Worker, Enable: false}, - // load balancer - "haproxy": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "library", Repo: "haproxy", Tag: "2.3", Group: kubekeyv1alpha2.Worker, Enable: kubeConf.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, - "kubevip": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "plndr", Repo: "kube-vip", Tag: "v0.5.0", Group: kubekeyv1alpha2.Master, Enable: kubeConf.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - // kata-deploy - "kata-deploy": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "kata-deploy", Tag: "stable", Group: kubekeyv1alpha2.Worker, Enable: kubeConf.Cluster.Kubernetes.EnableKataDeploy()}, - // node-feature-discovery - "node-feature-discovery": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "node-feature-discovery", Tag: "v0.10.0", Group: kubekeyv1alpha2.K8s, Enable: kubeConf.Cluster.Kubernetes.EnableNodeFeatureDiscovery()}, - } - - image = ImageList[name] - if kubeConf.Cluster.Registry.NamespaceOverride != "" { - image.NamespaceOverride = kubeConf.Cluster.Registry.NamespaceOverride - } - return image -} - -type SaveImages struct { - common.ArtifactAction -} - -func (s *SaveImages) Execute(runtime connector.Runtime) error { - auths := registry.DockerRegistryAuthEntries(s.Manifest.Spec.ManifestRegistry.Auths) - - dirName := filepath.Join(runtime.GetWorkDir(), common.Artifact, "images") - if err := coreutil.Mkdir(dirName); err != nil { - return errors.Wrapf(errors.WithStack(err), "mkdir %s failed", dirName) - } - for _, image := range s.Manifest.Spec.Images { - if err := validateImageName(image); err != nil { - return err - } - imageFullName := strings.Split(image, "/") - repo := imageFullName[0] - auth := new(registry.DockerRegistryEntry) - if v, ok := auths[repo]; ok { - auth = v - } - - srcName := fmt.Sprintf("docker://%s", image) - for _, platform := range s.Manifest.Spec.Arches { - arch, variant := ParseArchVariant(platform) - // placeholder - if variant != "" { - variant = "-" + variant - } - // Ex: - // oci:./kubekey/artifact/images:kubesphere:kube-apiserver:v1.21.5-amd64 - // oci:./kubekey/artifact/images:kubesphere:kube-apiserver:v1.21.5-arm-v7 - destName := fmt.Sprintf("oci:%s:%s:%s-%s%s", dirName, imageFullName[1], suffixImageName(imageFullName[2:]), arch, variant) - logger.Log.Infof("Source: %s", srcName) - logger.Log.Infof("Destination: %s", destName) - - o := &CopyImageOptions{ - srcImage: &srcImageOptions{ - imageName: srcName, - dockerImage: dockerImageOptions{ - arch: arch, - variant: variant, - os: "linux", - username: auth.Username, - password: auth.Password, - SkipTLSVerify: auth.SkipTLSVerify, - dockerCertPath: auth.CertsPath, - }, - }, - destImage: &destImageOptions{ - imageName: destName, - dockerImage: dockerImageOptions{ - arch: arch, - variant: variant, - os: "linux", - }, - }, - } - - if err := o.Copy(); err != nil { - return err - } - } - } - return nil -} - -type CopyImagesToRegistry struct { - common.KubeAction - ImagesPath string -} - -func (c *CopyImagesToRegistry) Execute(runtime connector.Runtime) error { - var imagesPath string - if c.ImagesPath != "" { - imagesPath = c.ImagesPath - } else { - imagesPath = filepath.Join(runtime.GetWorkDir(), "images") - } - - indexFile, err := os.ReadFile(filepath.Join(imagesPath, "index.json")) - if err != nil { - return errors.Errorf("read index.json failed: %s", err) - } - - index := NewIndex() - if err := json.Unmarshal(indexFile, index); err != nil { - return errors.Wrap(errors.WithStack(err), "unmarshal index.json failed: %s") - } - - auths := registry.DockerRegistryAuthEntries(c.KubeConf.Cluster.Registry.Auths) - - manifestList := make(map[string][]manifesttypes.ManifestEntry) - for _, m := range index.Manifests { - ref := m.Annotations.RefName - - // Ex: - // calico:cni:v3.20.0-amd64 - nameArr := strings.Split(ref, ":") - if len(nameArr) != 3 { - return errors.Errorf("invalid ref name: %s", ref) - } - - image := Image{ - RepoAddr: c.KubeConf.Cluster.Registry.PrivateRegistry, - Namespace: nameArr[0], - NamespaceOverride: c.KubeConf.Cluster.Registry.NamespaceOverride, - Repo: nameArr[1], - Tag: nameArr[2], - } - - uniqueImage, p := ParseImageWithArchTag(image.ImageName()) - entry := manifesttypes.ManifestEntry{ - Image: image.ImageName(), - Platform: p, - } - - skip := false - if v, ok := manifestList[uniqueImage]; ok { - // skip if the image already copied - for _, old := range v { - if reflect.DeepEqual(old, entry) { - skip = true - break - } - } - - if !skip { - v = append(v, entry) - manifestList[uniqueImage] = v - } - } else { - entryArr := make([]manifesttypes.ManifestEntry, 0) - manifestList[uniqueImage] = append(entryArr, entry) - } - - auth := new(registry.DockerRegistryEntry) - if config, ok := auths[c.KubeConf.Cluster.Registry.PrivateRegistry]; ok { - auth = config - } - - srcName := fmt.Sprintf("oci:%s:%s", imagesPath, ref) - destName := fmt.Sprintf("docker://%s", image.ImageName()) - logger.Log.Infof("Source: %s", srcName) - logger.Log.Infof("Destination: %s", destName) - - o := &CopyImageOptions{ - srcImage: &srcImageOptions{ - imageName: srcName, - dockerImage: dockerImageOptions{ - arch: p.Architecture, - variant: p.Variant, - os: "linux", - }, - }, - destImage: &destImageOptions{ - imageName: destName, - dockerImage: dockerImageOptions{ - arch: p.Architecture, - variant: p.Variant, - os: "linux", - username: auth.Username, - password: auth.Password, - SkipTLSVerify: auth.SkipTLSVerify, - dockerCertPath: auth.CertsPath, - }, - }, - } - - retry, maxRetry := 0, 5 - for ; retry < maxRetry; retry++ { - if err := o.Copy(); err == nil { - break - } - } - if retry >= maxRetry { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("copy image %s to %s failed, retry %d", srcName, destName, maxRetry)) - } - } - - c.ModuleCache.Set("manifestList", manifestList) - - return nil -} - -type PushManifest struct { - common.KubeAction -} - -func (p *PushManifest) Execute(_ connector.Runtime) error { - // make a multi-arch image - // push a manifest list to the private registry. - - v, ok := p.ModuleCache.Get("manifestList") - if !ok { - return errors.New("get manifest list failed by module cache") - } - list := v.(map[string][]manifesttypes.ManifestEntry) - - auths := registry.DockerRegistryAuthEntries(p.KubeConf.Cluster.Registry.Auths) - auth := new(registry.DockerRegistryEntry) - if _, ok := auths[p.KubeConf.Cluster.Registry.PrivateRegistry]; ok { - auth = auths[p.KubeConf.Cluster.Registry.PrivateRegistry] - } - - for imageName, platforms := range list { - manifestSpec := NewManifestSpec(imageName, platforms) - logger.Log.Debug(manifestSpec) - - logger.Log.Infof("Push multi-arch manifest list: %s", imageName) - // todo: the function can't support specify a certs dir - digest, length, err := manifestregistry.PushManifestList(auth.Username, auth.Password, manifestSpec, - false, true, auth.PlainHTTP, "") - if err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("push image %s multi-arch manifest failed", imageName)) - } - logger.Log.Infof("Digest: %s Length: %d", digest, length) - } - - return nil -} diff --git a/cmd/kk/pkg/images/utils.go b/cmd/kk/pkg/images/utils.go deleted file mode 100644 index 7e7a3ea6..00000000 --- a/cmd/kk/pkg/images/utils.go +++ /dev/null @@ -1,190 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "fmt" - "strings" - - "github.com/containerd/containerd/platforms" - "github.com/containers/image/v5/types" - manifesttypes "github.com/estesp/manifest-tool/v2/pkg/types" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -var defaultUserAgent = "kubekey" - -type dockerImageOptions struct { - arch string - os string - variant string - username string - password string - dockerCertPath string - SkipTLSVerify bool -} - -func (d *dockerImageOptions) systemContext() *types.SystemContext { - ctx := &types.SystemContext{ - ArchitectureChoice: d.arch, - OSChoice: d.os, - VariantChoice: d.variant, - DockerRegistryUserAgent: defaultUserAgent, - DockerInsecureSkipTLSVerify: types.NewOptionalBool(d.SkipTLSVerify), - } - return ctx -} - -type srcImageOptions struct { - dockerImage dockerImageOptions - imageName string - sharedBlobDir string -} - -func (s *srcImageOptions) systemContext() *types.SystemContext { - ctx := s.dockerImage.systemContext() - ctx.DockerCertPath = s.dockerImage.dockerCertPath - ctx.OCISharedBlobDirPath = s.sharedBlobDir - ctx.DockerAuthConfig = &types.DockerAuthConfig{ - Username: s.dockerImage.username, - Password: s.dockerImage.password, - } - - return ctx -} - -type destImageOptions struct { - dockerImage dockerImageOptions - imageName string -} - -func (d *destImageOptions) systemContext() *types.SystemContext { - ctx := d.dockerImage.systemContext() - ctx.DockerCertPath = d.dockerImage.dockerCertPath - ctx.DockerAuthConfig = &types.DockerAuthConfig{ - Username: d.dockerImage.username, - Password: d.dockerImage.password, - } - - return ctx -} - -// ParseArchVariant -// Ex: -// amd64 returns amd64, "" -// arm/v8 returns arm, v8 -func ParseArchVariant(platform string) (string, string) { - osArchArr := strings.Split(platform, "/") - - variant := "" - arch := osArchArr[0] - if len(osArchArr) > 1 { - variant = osArchArr[1] - } - return arch, variant -} - -func ParseImageWithArchTag(ref string) (string, ocispec.Platform) { - n := strings.LastIndex(ref, "-") - if n < 0 { - logger.Log.Fatalf("get arch or variant index failed: %s", ref) - } - archOrVariant := ref[n+1:] - - // try to parse the arch-only case - specifier := fmt.Sprintf("linux/%s", archOrVariant) - if p, err := platforms.Parse(specifier); err == nil && isKnownArch(p.Architecture) { - return ref[:n], p - } - - archStr := ref[:n] - a := strings.LastIndex(archStr, "-") - if a < 0 { - logger.Log.Fatalf("get arch index failed: %s", ref) - } - arch := archStr[a+1:] - - // parse the case where both arch and variant exist - specifier = fmt.Sprintf("linux/%s/%s", arch, archOrVariant) - p, err := platforms.Parse(specifier) - if err != nil { - logger.Log.Fatalf("parse image %s failed: %s", ref, err.Error()) - } - - return ref[:a], p -} - -func isKnownArch(arch string) bool { - switch arch { - case "386", "amd64", "amd64p32", "arm", "armbe", "arm64", "arm64be", "ppc64", "ppc64le", "loong64", "mips", "mipsle", "mips64", "mips64le", "mips64p32", "mips64p32le", "ppc", "riscv", "riscv64", "s390", "s390x", "sparc", "sparc64", "wasm": - return true - } - return false -} - -// ParseImageTag -// Get a repos name and returns the right reposName + tag -// The tag can be confusing because of a port in a repository name. -// -// Ex: localhost.localdomain:5000/samalba/hipache:latest -func ParseImageTag(repos string) (string, string) { - n := strings.LastIndex(repos, ":") - if n < 0 { - return repos, "" - } - if tag := repos[n+1:]; !strings.Contains(tag, "/") { - return repos[:n], tag - } - return repos, "" -} - -func NewManifestSpec(image string, entries []manifesttypes.ManifestEntry) manifesttypes.YAMLInput { - var srcImages []manifesttypes.ManifestEntry - - for _, e := range entries { - srcImages = append(srcImages, manifesttypes.ManifestEntry{ - Image: e.Image, - Platform: e.Platform, - }) - } - - return manifesttypes.YAMLInput{ - Image: image, - Manifests: srcImages, - } -} - -func validateImageName(imageFullName string) error { - image := strings.Split(imageFullName, "/") - if len(image) < 3 { - return errors.Errorf("image %s is invalid, image PATH need contain at least two slash-separated", imageFullName) - } - if len(strings.Split(image[len(image)-1], ":")) != 2 { - return errors.Errorf(`image %s is invalid, image PATH need contain ":"`, imageFullName) - } - return nil -} - -func suffixImageName(imageFullName []string) string { - if len(imageFullName) >= 2 { - return strings.Join(imageFullName, "/") - } - return imageFullName[0] -} diff --git a/cmd/kk/pkg/images/utils_test.go b/cmd/kk/pkg/images/utils_test.go deleted file mode 100644 index d393e8ff..00000000 --- a/cmd/kk/pkg/images/utils_test.go +++ /dev/null @@ -1,172 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "reflect" - "testing" - - v1 "github.com/opencontainers/image-spec/specs-go/v1" -) - -func TestParseArchVariant(t *testing.T) { - type args struct { - platform string - } - tests := []struct { - name string - args args - want string - want1 string - }{ - { - name: "test1", - args: args{ - platform: "amd64", - }, - want: "amd64", - want1: "", - }, - { - name: "test2", - args: args{ - platform: "arm/v8", - }, - want: "arm", - want1: "v8", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, got1 := ParseArchVariant(tt.args.platform) - if got != tt.want { - t.Errorf("ParseArchVariant() got = %v, want %v", got, tt.want) - } - if got1 != tt.want1 { - t.Errorf("ParseArchVariant() got1 = %v, want %v", got1, tt.want1) - } - }) - } -} - -func TestParseRepositoryTag(t *testing.T) { - type args struct { - repos string - } - tests := []struct { - name string - args args - want string - want1 string - }{ - { - name: "test1", - args: args{ - repos: "k8s.gcr.io/kube-apiserver-amd64:v1.16.3", - }, - want: "k8s.gcr.io/kube-apiserver-amd64", - want1: "v1.16.3", - }, - { - name: "test2", - args: args{ - repos: "docker.io/kubesphere/kube-apiserver:v1.16.3", - }, - want: "docker.io/kubesphere/kube-apiserver", - want1: "v1.16.3", - }, - { - name: "test3", - args: args{ - repos: "kube-apiserver:v1.16.3", - }, - want: "kube-apiserver", - want1: "v1.16.3", - }, - { - name: "test4", - args: args{ - repos: "calico:cni:v3.20.0-armv7", - }, - want: "calico:cni", - want1: "v3.20.0-armv7", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, got1 := ParseImageTag(tt.args.repos) - if got != tt.want { - t.Errorf("ParseImageTag() got = %v, want %v", got, tt.want) - } - if got1 != tt.want1 { - t.Errorf("ParseImageTag() got1 = %v, want %v", got1, tt.want1) - } - }) - } -} - -func TestParseImageWithArchTag(t *testing.T) { - tests := []struct { - name string - ref string - want1 string - want2 v1.Platform - }{ - { - name: "t1", - ref: "kube-apiserver:v1.21.5-amd64", - want1: "kube-apiserver:v1.21.5", - want2: v1.Platform{ - OS: "linux", - Architecture: "amd64", - Variant: "", - }, - }, - { - name: "t1", - ref: "kube-apiserver:v1.21.5-amd64", - want1: "kube-apiserver:v1.21.5", - want2: v1.Platform{ - OS: "linux", - Architecture: "amd64", - }, - }, - { - name: "t2", - ref: "kube-apiserver:v1.21.5-arm-v7", - want1: "kube-apiserver:v1.21.5", - want2: v1.Platform{ - OS: "linux", - Architecture: "arm", - Variant: "v7", - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, got1 := ParseImageWithArchTag(tt.ref) - - if got != tt.want1 { - t.Errorf("ParseImageTag() got = %v, want %v", got, tt.want1) - } - - if !reflect.DeepEqual(got1, tt.want2) { - t.Errorf("ParseImageWithArchTag() = %v, want %v", got, tt.want2) - } - }) - } -} diff --git a/cmd/kk/pkg/k3s/k3s_status.go b/cmd/kk/pkg/k3s/k3s_status.go deleted file mode 100644 index f93810dd..00000000 --- a/cmd/kk/pkg/k3s/k3s_status.go +++ /dev/null @@ -1,124 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k3s - -import ( - "fmt" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type K3sStatus struct { - Version string - ClusterInfo string - NodeToken string - KubeConfig string - NodesInfo map[string]string -} - -func NewK3sStatus() *K3sStatus { - return &K3sStatus{NodesInfo: make(map[string]string)} -} - -func (k *K3sStatus) SearchVersion(runtime connector.Runtime) error { - cmd := "k3s --version | grep 'k3s' | awk '{print $3}'" - if output, err := runtime.GetRunner().Cmd(cmd, true); err != nil { - return errors.Wrap(errors.WithStack(err), "search current version failed") - } else { - k.Version = output - } - return nil -} - -func (k *K3sStatus) SearchKubeConfig(runtime connector.Runtime) error { - kubeCfgCmd := "cat /etc/rancher/k3s/k3s.yaml" - if kubeConfigStr, err := runtime.GetRunner().SudoCmd(kubeCfgCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "search cluster kubeconfig failed") - } else { - k.KubeConfig = kubeConfigStr - } - return nil -} - -func (k *K3sStatus) SearchNodeToken(runtime connector.Runtime) error { - nodeTokenBase64Cmd := "cat /var/lib/rancher/k3s/server/node-token" - output, err := runtime.GetRunner().SudoCmd(nodeTokenBase64Cmd, true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get cluster node token failed") - } - k.NodeToken = output - return nil -} - -func (k *K3sStatus) SearchInfo(runtime connector.Runtime) error { - output, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl --no-headers=true get nodes -o custom-columns=:metadata.name,:status.nodeInfo.kubeletVersion,:status.addresses", - true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get k3s cluster info failed") - } - k.ClusterInfo = output - return nil -} - -func (k *K3sStatus) SearchNodesInfo(_ connector.Runtime) error { - ipv4Regexp, err := regexp.Compile(common.IPv4Regexp) - if err != nil { - return err - } - ipv6Regexp, err := regexp.Compile(common.IPv6Regexp) - if err != nil { - return err - } - tmp := strings.Split(k.ClusterInfo, "\r\n") - if len(tmp) >= 1 { - for i := 0; i < len(tmp); i++ { - if ipv4 := ipv4Regexp.FindStringSubmatch(tmp[i]); len(ipv4) != 0 { - k.NodesInfo[ipv4[0]] = ipv4[0] - } - if ipv6 := ipv6Regexp.FindStringSubmatch(tmp[i]); len(ipv6) != 0 { - k.NodesInfo[ipv6[0]] = ipv6[0] - } - if len(strings.Fields(tmp[i])) > 3 { - k.NodesInfo[strings.Fields(tmp[i])[0]] = strings.Fields(tmp[i])[1] - } else { - k.NodesInfo[strings.Fields(tmp[i])[0]] = "" - } - } - } - return nil -} - -func (k *K3sStatus) LoadKubeConfig(runtime connector.Runtime, kubeConf *common.KubeConf) error { - kubeConfigPath := filepath.Join(runtime.GetWorkDir(), fmt.Sprintf("config-%s", runtime.GetObjName())) - - oldServer := "server: https://127.0.0.1:6443" - newServer := fmt.Sprintf("server: https://%s:%d", kubeConf.Cluster.ControlPlaneEndpoint.Address, kubeConf.Cluster.ControlPlaneEndpoint.Port) - newKubeConfigStr := strings.Replace(k.KubeConfig, oldServer, newServer, -1) - - if err := os.WriteFile(kubeConfigPath, []byte(newKubeConfigStr), 0644); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/k3s/module.go b/cmd/kk/pkg/k3s/module.go deleted file mode 100644 index 9520d88e..00000000 --- a/cmd/kk/pkg/k3s/module.go +++ /dev/null @@ -1,343 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k3s - -import ( - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k3s/templates" -) - -type StatusModule struct { - common.KubeModule -} - -func (s *StatusModule) Init() { - s.Name = "StatusModule" - s.Desc = "Get cluster status" - - cluster := NewK3sStatus() - s.PipelineCache.GetOrSet(common.ClusterStatus, cluster) - - clusterStatus := &task.RemoteTask{ - Name: "GetClusterStatus", - Desc: "Get k3s cluster status", - Hosts: s.Runtime.GetHostsByRole(common.Master), - Action: new(GetClusterStatus), - Parallel: false, - } - - s.Tasks = []task.Interface{ - clusterStatus, - } -} - -type InstallKubeBinariesModule struct { - common.KubeModule -} - -func (i *InstallKubeBinariesModule) Init() { - i.Name = "InstallKubeBinariesModule" - i.Desc = "Install k3s cluster" - - syncBinary := &task.RemoteTask{ - Name: "SyncKubeBinary", - Desc: "Synchronize k3s binaries", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: new(SyncKubeBinary), - Parallel: true, - Retry: 2, - } - - killAllScript := &task.RemoteTask{ - Name: "GenerateK3sKillAllScript", - Desc: "Generate k3s killall.sh script", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: &action.Template{ - Template: templates.K3sKillallScript, - Dst: filepath.Join("/usr/local/bin", templates.K3sKillallScript.Name()), - }, - Parallel: true, - Retry: 2, - } - - uninstallScript := &task.RemoteTask{ - Name: "GenerateK3sUninstallScript", - Desc: "Generate k3s uninstall script", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: &action.Template{ - Template: templates.K3sUninstallScript, - Dst: filepath.Join("/usr/local/bin", templates.K3sUninstallScript.Name()), - }, - Parallel: true, - Retry: 2, - } - - chmod := &task.RemoteTask{ - Name: "ChmodScript", - Desc: "Chmod +x k3s script ", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: new(ChmodScript), - Parallel: true, - Retry: 2, - } - - i.Tasks = []task.Interface{ - syncBinary, - killAllScript, - uninstallScript, - chmod, - } -} - -type InitClusterModule struct { - common.KubeModule -} - -func (i *InitClusterModule) Init() { - i.Name = "K3sInitClusterModule" - i.Desc = "Init k3s cluster" - - k3sService := &task.RemoteTask{ - Name: "GenerateK3sService", - Desc: "Generate k3s Service", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(GenerateK3sService), - Parallel: true, - } - - k3sEnv := &task.RemoteTask{ - Name: "GenerateK3sServiceEnv", - Desc: "Generate k3s service env", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(GenerateK3sServiceEnv), - Parallel: true, - } - - k3sRegistryConfig := &task.RemoteTask{ - Name: "GenerateK3sRegistryConfig", - Desc: "Generate k3s registry config", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - &UsePrivateRegstry{Not: false}, - }, - Action: new(GenerateK3sRegistryConfig), - Parallel: true, - } - - enableK3s := &task.RemoteTask{ - Name: "EnableK3sService", - Desc: "Enable k3s service", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(EnableK3sService), - Parallel: true, - } - - copyKubeConfig := &task.RemoteTask{ - Name: "CopyKubeConfig", - Desc: "Copy k3s.yaml to ~/.kube/config", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(CopyK3sKubeConfig), - Parallel: true, - } - - addMasterTaint := &task.RemoteTask{ - Name: "AddMasterTaint", - Desc: "Add master taint", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - &common.IsWorker{Not: true}, - }, - Action: new(AddMasterTaint), - Parallel: true, - Retry: 5, - } - - i.Tasks = []task.Interface{ - k3sService, - k3sEnv, - k3sRegistryConfig, - enableK3s, - copyKubeConfig, - addMasterTaint, - } -} - -type JoinNodesModule struct { - common.KubeModule -} - -func (j *JoinNodesModule) Init() { - j.Name = "K3sJoinNodesModule" - j.Desc = "Join k3s nodes" - - k3sService := &task.RemoteTask{ - Name: "GenerateK3sService", - Desc: "Generate k3s Service", - Hosts: j.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(GenerateK3sService), - Parallel: true, - } - - k3sEnv := &task.RemoteTask{ - Name: "GenerateK3sServiceEnv", - Desc: "Generate k3s service env", - Hosts: j.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(GenerateK3sServiceEnv), - Parallel: true, - } - - k3sRegistryConfig := &task.RemoteTask{ - Name: "GenerateK3sRegistryConfig", - Desc: "Generate k3s registry config", - Hosts: j.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - &UsePrivateRegstry{Not: false}, - }, - Action: new(GenerateK3sRegistryConfig), - Parallel: true, - } - - enableK3s := &task.RemoteTask{ - Name: "EnableK3sService", - Desc: "Enable k3s service", - Hosts: j.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(EnableK3sService), - Parallel: true, - } - - copyKubeConfigForMaster := &task.RemoteTask{ - Name: "CopyKubeConfig", - Desc: "Copy k3s.yaml to ~/.kube/config", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(CopyK3sKubeConfig), - Parallel: true, - } - - addMasterTaint := &task.RemoteTask{ - Name: "AddMasterTaint", - Desc: "Add master taint", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - &common.IsWorker{Not: true}, - }, - Action: new(AddMasterTaint), - Parallel: true, - Retry: 5, - } - - addWorkerLabelToNode := &task.RemoteTask{ - Name: "addWorkerLabelToNode", - Desc: "Add worker label to all nodes", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(AddWorkerLabel), - Retry: 3, - } - - j.Tasks = []task.Interface{ - k3sService, - k3sEnv, - k3sRegistryConfig, - enableK3s, - copyKubeConfigForMaster, - addMasterTaint, - addWorkerLabelToNode, - } -} - -type DeleteClusterModule struct { - common.KubeModule -} - -func (d *DeleteClusterModule) Init() { - d.Name = "DeleteClusterModule" - d.Desc = "Delete k3s cluster" - - execScript := &task.RemoteTask{ - Name: "ExecUninstallScript", - Desc: "Exec k3s uninstall script", - Hosts: d.Runtime.GetHostsByRole(common.K8s), - Action: new(ExecUninstallScript), - Parallel: true, - } - - d.Tasks = []task.Interface{ - execScript, - } -} - -type SaveKubeConfigModule struct { - common.KubeModule -} - -func (s *SaveKubeConfigModule) Init() { - s.Name = "SaveKubeConfigModule" - s.Desc = "Save kube config file as a configmap" - - save := &task.LocalTask{ - Name: "SaveKubeConfig", - Desc: "Save kube config as a configmap", - Action: new(SaveKubeConfig), - } - - s.Tasks = []task.Interface{ - save, - } -} diff --git a/cmd/kk/pkg/k3s/prepares.go b/cmd/kk/pkg/k3s/prepares.go deleted file mode 100644 index 0ff3bd9d..00000000 --- a/cmd/kk/pkg/k3s/prepares.go +++ /dev/null @@ -1,71 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k3s - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type NodeInCluster struct { - common.KubePrepare - Not bool -} - -func (n *NodeInCluster) PreCheck(runtime connector.Runtime) (bool, error) { - host := runtime.RemoteHost() - if v, ok := n.PipelineCache.Get(common.ClusterStatus); ok { - cluster := v.(*K3sStatus) - var versionOk bool - if res, ok := cluster.NodesInfo[host.GetName()]; ok && res != "" { - versionOk = true - } - _, ipOk := cluster.NodesInfo[host.GetInternalAddress()] - if n.Not { - return !(versionOk || ipOk), nil - } - return versionOk || ipOk, nil - } - return false, errors.New("get k3s cluster status by pipeline cache failed") -} - -type ClusterIsExist struct { - common.KubePrepare - Not bool -} - -func (c *ClusterIsExist) PreCheck(_ connector.Runtime) (bool, error) { - if exist, ok := c.PipelineCache.GetMustBool(common.ClusterExist); ok { - if c.Not { - return !exist, nil - } - return exist, nil - } else { - return false, errors.New("get k3s cluster status by pipeline cache failed") - } -} - -type UsePrivateRegstry struct { - common.KubePrepare - Not bool -} - -func (c *UsePrivateRegstry) PreCheck(_ connector.Runtime) (bool, error) { - return c.KubeConf.Cluster.Registry.PrivateRegistry != "", nil -} diff --git a/cmd/kk/pkg/k3s/tasks.go b/cmd/kk/pkg/k3s/tasks.go deleted file mode 100644 index 9b68c81d..00000000 --- a/cmd/kk/pkg/k3s/tasks.go +++ /dev/null @@ -1,531 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k3s - -import ( - "context" - "encoding/base64" - "fmt" - "path/filepath" - "strings" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - versionutil "k8s.io/apimachinery/pkg/util/version" - kube "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k3s/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -type GetClusterStatus struct { - common.KubeAction -} - -func (g *GetClusterStatus) Execute(runtime connector.Runtime) error { - exist, err := runtime.GetRunner().FileExist("/etc/systemd/system/k3s.service") - if err != nil { - return err - } - - if !exist { - g.PipelineCache.Set(common.ClusterExist, false) - return nil - } else { - g.PipelineCache.Set(common.ClusterExist, true) - - if v, ok := g.PipelineCache.Get(common.ClusterStatus); ok { - cluster := v.(*K3sStatus) - if err := cluster.SearchVersion(runtime); err != nil { - return err - } - if err := cluster.SearchKubeConfig(runtime); err != nil { - return err - } - if err := cluster.LoadKubeConfig(runtime, g.KubeConf); err != nil { - return err - } - if err := cluster.SearchNodeToken(runtime); err != nil { - return err - } - if err := cluster.SearchInfo(runtime); err != nil { - return err - } - if err := cluster.SearchNodesInfo(runtime); err != nil { - return err - } - g.PipelineCache.Set(common.ClusterStatus, cluster) - } else { - return errors.New("get k3s cluster status by pipeline cache failed") - } - } - return nil -} - -type SyncKubeBinary struct { - common.KubeAction -} - -func (s *SyncKubeBinary) Execute(runtime connector.Runtime) error { - binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - if err := SyncKubeBinaries(s, runtime, binariesMap); err != nil { - return err - } - return nil -} - -// SyncKubeBinaries is used to sync kubernetes' binaries to each node. -func SyncKubeBinaries(s *SyncKubeBinary, runtime connector.Runtime, binariesMap map[string]*files.KubeBinary) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binaryList := []string{"k3s", "helm", "kubecni"} - if s.KubeConf.Cluster.Network.Plugin == "calico" { - binaryList = append(binaryList, "calicoctl") - } - for _, name := range binaryList { - binary, ok := binariesMap[name] - if !ok { - return fmt.Errorf("get kube binary %s info failed: no such key", name) - } - - fileName := binary.FileName - switch name { - case "kubecni": - dst := filepath.Join(common.TmpDir, fileName) - if err := runtime.GetRunner().Scp(binary.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed")) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("tar -zxf %s -C /opt/cni/bin", dst), false); err != nil { - return err - } - default: - dst := filepath.Join(common.BinDir, fileName) - if err := runtime.GetRunner().SudoScp(binary.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed")) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s", dst), false); err != nil { - return err - } - } - } - - binaries := []string{"kubectl", "crictl", "ctr"} - var createLinkCMDs []string - for _, binary := range binaries { - createLinkCMDs = append(createLinkCMDs, fmt.Sprintf("ln -snf /usr/local/bin/k3s /usr/local/bin/%s", binary)) - } - if _, err := runtime.GetRunner().SudoCmd(strings.Join(createLinkCMDs, " && "), false); err != nil { - return errors.Wrap(errors.WithStack(err), "create ctl tool link failed") - } - - return nil -} - -type ChmodScript struct { - common.KubeAction -} - -func (c *ChmodScript) Execute(runtime connector.Runtime) error { - killAllScript := filepath.Join("/usr/local/bin", templates.K3sKillallScript.Name()) - uninstallScript := filepath.Join("/usr/local/bin", templates.K3sUninstallScript.Name()) - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s", killAllScript), - false); err != nil { - return err - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s", uninstallScript), - false); err != nil { - return err - } - return nil -} - -type GenerateK3sService struct { - common.KubeAction -} - -func (g *GenerateK3sService) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - var server string - if !host.IsRole(common.Master) { - server = fmt.Sprintf("https://%s:%d", g.KubeConf.Cluster.ControlPlaneEndpoint.Domain, g.KubeConf.Cluster.ControlPlaneEndpoint.Port) - } - - defaultKubeletArs := map[string]string{ - "cni-conf-dir": "/etc/cni/net.d", - "cni-bin-dir": "/opt/cni/bin", - "kube-reserved": "cpu=200m,memory=250Mi,ephemeral-storage=1Gi", - "system-reserved": "cpu=200m,memory=250Mi,ephemeral-storage=1Gi", - "eviction-hard": "memory.available<5%,nodefs.available<10%", - } - defaultKubeProxyArgs := map[string]string{ - "proxy-mode": "ipvs", - } - - kubeApiserverArgs, _ := util.GetArgs(map[string]string{}, g.KubeConf.Cluster.Kubernetes.ApiServerArgs) - kubeControllerManager, _ := util.GetArgs(map[string]string{ - "pod-eviction-timeout": "3m0s", - "terminated-pod-gc-threshold": "5", - }, g.KubeConf.Cluster.Kubernetes.ControllerManagerArgs) - kubeSchedulerArgs, _ := util.GetArgs(map[string]string{}, g.KubeConf.Cluster.Kubernetes.SchedulerArgs) - kubeletArgs, _ := util.GetArgs(defaultKubeletArs, g.KubeConf.Cluster.Kubernetes.KubeletArgs) - kubeProxyArgs, _ := util.GetArgs(defaultKubeProxyArgs, g.KubeConf.Cluster.Kubernetes.KubeProxyArgs) - - templateAction := action.Template{ - Template: templates.K3sService, - Dst: filepath.Join("/etc/systemd/system/", templates.K3sService.Name()), - Data: util.Data{ - "Server": server, - "IsMaster": host.IsRole(common.Master), - "IsDockerRuntime": g.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker, - "ContainerRuntimeEndpoint": g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, - "NodeIP": host.GetInternalAddress(), - "HostName": host.GetName(), - "PodSubnet": g.KubeConf.Cluster.Network.KubePodsCIDR, - "ServiceSubnet": g.KubeConf.Cluster.Network.KubeServiceCIDR, - "ClusterDns": g.KubeConf.Cluster.CorednsClusterIP(), - "CertSANs": g.KubeConf.Cluster.GenerateCertSANs(), - "PauseImage": images.GetImage(runtime, g.KubeConf, "pause").ImageName(), - "ApiserverArgs": kubeApiserverArgs, - "ControllerManager": kubeControllerManager, - "SchedulerArgs": kubeSchedulerArgs, - "KubeletArgs": kubeletArgs, - "KubeProxyArgs": kubeProxyArgs, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type GenerateK3sServiceEnv struct { - common.KubeAction -} - -func (g *GenerateK3sServiceEnv) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - clusterStatus, ok := g.PipelineCache.Get(common.ClusterStatus) - if !ok { - return errors.New("get cluster status by pipeline cache failed") - } - cluster := clusterStatus.(*K3sStatus) - - var externalEtcd kubekeyapiv1alpha2.ExternalEtcd - var endpointsList []string - var externalEtcdEndpoints, token string - - switch g.KubeConf.Cluster.Etcd.Type { - case kubekeyapiv1alpha2.External: - externalEtcd.Endpoints = g.KubeConf.Cluster.Etcd.External.Endpoints - - if len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 { - externalEtcd.CAFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CAFile)) - externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CertFile)) - externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.KeyFile)) - } - default: - for _, node := range runtime.GetHostsByRole(common.ETCD) { - endpoint := fmt.Sprintf("https://%s:%s", node.GetInternalAddress(), kubekeyapiv1alpha2.DefaultEtcdPort) - endpointsList = append(endpointsList, endpoint) - } - externalEtcd.Endpoints = endpointsList - - externalEtcd.CAFile = "/etc/ssl/etcd/ssl/ca.pem" - externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s.pem", runtime.GetHostsByRole(common.Master)[0].GetName()) - externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s-key.pem", runtime.GetHostsByRole(common.Master)[0].GetName()) - } - - externalEtcdEndpoints = strings.Join(externalEtcd.Endpoints, ",") - - v121 := versionutil.MustParseSemantic("v1.21.0") - atLeast := versionutil.MustParseSemantic(g.KubeConf.Cluster.Kubernetes.Version).AtLeast(v121) - if atLeast { - token = cluster.NodeToken - } else { - if !host.IsRole(common.Master) { - token = cluster.NodeToken - } - } - - templateAction := action.Template{ - Template: templates.K3sServiceEnv, - Dst: filepath.Join("/etc/systemd/system/", templates.K3sServiceEnv.Name()), - Data: util.Data{ - "DataStoreEndPoint": externalEtcdEndpoints, - "DataStoreCaFile": externalEtcd.CAFile, - "DataStoreCertFile": externalEtcd.CertFile, - "DataStoreKeyFile": externalEtcd.KeyFile, - "IsMaster": host.IsRole(common.Master), - "Token": token, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type EnableK3sService struct { - common.KubeAction -} - -func (e *EnableK3sService) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && systemctl enable --now k3s", - false); err != nil { - return errors.Wrap(errors.WithStack(err), "enable k3s failed") - } - return nil -} - -type CopyK3sKubeConfig struct { - common.KubeAction -} - -func (c *CopyK3sKubeConfig) Execute(runtime connector.Runtime) error { - createConfigDirCmd := "mkdir -p /root/.kube && mkdir -p $HOME/.kube" - getKubeConfigCmd := "cp -f /etc/rancher/k3s/k3s.yaml /root/.kube/config" - - cmd := strings.Join([]string{createConfigDirCmd, getKubeConfigCmd}, " && ") - if _, err := runtime.GetRunner().SudoCmd(cmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "copy k3s kube config failed") - } - - userMkdir := "mkdir -p $HOME/.kube" - if _, err := runtime.GetRunner().Cmd(userMkdir, false); err != nil { - return errors.Wrap(errors.WithStack(err), "user mkdir $HOME/.kube failed") - } - - userCopyKubeConfig := "cp -f /etc/rancher/k3s/k3s.yaml $HOME/.kube/config" - if _, err := runtime.GetRunner().SudoCmd(userCopyKubeConfig, false); err != nil { - return errors.Wrap(errors.WithStack(err), "user copy /etc/rancher/k3s/k3s.yaml to $HOME/.kube/config failed") - } - - userId, err := runtime.GetRunner().Cmd("echo $(id -u)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user id failed") - } - - userGroupId, err := runtime.GetRunner().Cmd("echo $(id -g)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user group id failed") - } - - chownKubeConfig := fmt.Sprintf("chown -R %s:%s $HOME/.kube", userId, userGroupId) - if _, err := runtime.GetRunner().SudoCmd(chownKubeConfig, false); err != nil { - return errors.Wrap(errors.WithStack(err), "chown user kube config failed") - } - return nil -} - -type AddMasterTaint struct { - common.KubeAction -} - -func (a *AddMasterTaint) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - cmd := fmt.Sprintf( - "/usr/local/bin/kubectl taint nodes %s node-role.kubernetes.io/master=effect:NoSchedule --overwrite", - host.GetName()) - - if _, err := runtime.GetRunner().SudoCmd(cmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "add master NoSchedule taint failed") - } - return nil -} - -type AddWorkerLabel struct { - common.KubeAction -} - -func (a *AddWorkerLabel) Execute(runtime connector.Runtime) error { - for _, host := range runtime.GetAllHosts() { - if host.IsRole(common.Worker) { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "/usr/local/bin/kubectl label --overwrite node %s node-role.kubernetes.io/worker=", - host.GetName()), true); err != nil { - return errors.Wrap(errors.WithStack(err), "add worker label failed") - } - } - } - - return nil -} - -type ExecUninstallScript struct { - common.KubeAction -} - -func (e *ExecUninstallScript) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && /usr/local/bin/k3s-killall.sh", - true); err != nil { - return errors.Wrap(errors.WithStack(err), "add master NoSchedule taint failed") - } - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && /usr/local/bin/k3s-uninstall.sh", - true); err != nil { - return errors.Wrap(errors.WithStack(err), "add master NoSchedule taint failed") - } - return nil -} - -type SaveKubeConfig struct { - common.KubeAction -} - -func (s *SaveKubeConfig) Execute(_ connector.Runtime) error { - status, ok := s.PipelineCache.Get(common.ClusterStatus) - if !ok { - return errors.New("get kubernetes status failed by pipeline cache") - } - cluster := status.(*K3sStatus) - - oldServer := fmt.Sprintf("https://%s:%d", s.KubeConf.Cluster.ControlPlaneEndpoint.Domain, s.KubeConf.Cluster.ControlPlaneEndpoint.Port) - newServer := fmt.Sprintf("https://%s:%d", s.KubeConf.Cluster.ControlPlaneEndpoint.Address, s.KubeConf.Cluster.ControlPlaneEndpoint.Port) - newKubeConfigStr := strings.Replace(cluster.KubeConfig, oldServer, newServer, -1) - kubeConfigBase64 := base64.StdEncoding.EncodeToString([]byte(newKubeConfigStr)) - - config, err := clientcmd.NewClientConfigFromBytes([]byte(newKubeConfigStr)) - if err != nil { - return err - } - restConfig, err := config.ClientConfig() - if err != nil { - return err - } - clientsetForCluster, err := kube.NewForConfig(restConfig) - if err != nil { - return err - } - - namespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubekey-system", - }, - } - if _, err := clientsetForCluster. - CoreV1(). - Namespaces(). - Create(context.TODO(), namespace, metav1.CreateOptions{}); err != nil { - return err - } - - cm := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-kubeconfig", s.KubeConf.ClusterName), - }, - Data: map[string]string{ - "kubeconfig": kubeConfigBase64, - }, - } - - if _, err := clientsetForCluster. - CoreV1(). - ConfigMaps("kubekey-system"). - Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil { - return err - } - return nil -} - -type GenerateK3sRegistryConfig struct { - common.KubeAction -} - -func (g *GenerateK3sRegistryConfig) Execute(runtime connector.Runtime) error { - endpointPrefix := "https://" - dockerioMirror := registry.Mirror{} - registryConfigs := map[string]registry.RegistryConfig{} - - auths := registry.DockerRegistryAuthEntries(g.KubeConf.Cluster.Registry.Auths) - for k, v := range auths { - if k == g.KubeConf.Cluster.Registry.PrivateRegistry && v.PlainHTTP { - endpointPrefix = "http://" - } - } - - dockerioMirror.Endpoints = []string{fmt.Sprintf("%s%s", endpointPrefix, g.KubeConf.Cluster.Registry.PrivateRegistry)} - - if g.KubeConf.Cluster.Registry.NamespaceOverride != "" { - dockerioMirror.Rewrites = map[string]string{ - "^rancher/(.*)": fmt.Sprintf("%s/$1", g.KubeConf.Cluster.Registry.NamespaceOverride), - } - } - - for k, v := range auths { - registryConfigs[k] = registry.RegistryConfig{ - Auth: ®istry.AuthConfig{ - Username: v.Username, - Password: v.Password, - }, - TLS: ®istry.TLSConfig{ - CAFile: v.CAFile, - CertFile: v.CertFile, - KeyFile: v.KeyFile, - InsecureSkipVerify: v.SkipTLSVerify, - }, - } - } - - _, ok := registryConfigs[g.KubeConf.Cluster.Registry.PrivateRegistry] - - if !ok { - registryConfigs[g.KubeConf.Cluster.Registry.PrivateRegistry] = registry.RegistryConfig{TLS: ®istry.TLSConfig{InsecureSkipVerify: true}} - } - - k3sRegistries := registry.Registry{ - Mirrors: map[string]registry.Mirror{"docker.io": dockerioMirror}, - Configs: registryConfigs, - } - - templateAction := action.Template{ - Template: templates.K3sRegistryConfigTempl, - Dst: filepath.Join("/etc/rancher/k3s", templates.K3sRegistryConfigTempl.Name()), - Data: util.Data{ - "Registries": k3sRegistries, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/k3s/templates/k3sRegistryConfig.go b/cmd/kk/pkg/k3s/templates/k3sRegistryConfig.go deleted file mode 100644 index cd5266d8..00000000 --- a/cmd/kk/pkg/k3s/templates/k3sRegistryConfig.go +++ /dev/null @@ -1,32 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -var ( - funcMap = template.FuncMap{"toYaml": utils.ToYAML, "indent": utils.Indent} - // k3sRegistryConfigTempl defines the template of k3s' registry. - K3sRegistryConfigTempl = template.Must(template.New("registries.yaml").Funcs(funcMap).Parse( - dedent.Dedent(`{{ toYaml .Registries }}`))) -) diff --git a/cmd/kk/pkg/k3s/templates/k3sService.go b/cmd/kk/pkg/k3s/templates/k3sService.go deleted file mode 100644 index b43a209c..00000000 --- a/cmd/kk/pkg/k3s/templates/k3sService.go +++ /dev/null @@ -1,59 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var ( - // K3sService defines the template of kubelet service for systemd. - K3sService = template.Must(template.New("k3s.service").Parse( - dedent.Dedent(`[Unit] -Description=Lightweight Kubernetes -Documentation=https://k3s.io -Wants=network-online.target -After=network-online.target - -[Install] -WantedBy=multi-user.target - -[Service] -Type=notify -EnvironmentFile=/etc/systemd/system/k3s.service.env -{{ if .IsMaster }} -Environment="K3S_ARGS={{ range .CertSANs }} --tls-san={{ . }}{{- end }} {{ range .ApiserverArgs }} --kube-apiserver-arg={{ . }}{{- end }} {{ range .ControllerManager }} --kube-controller-manager-arg={{ . }}{{- end }} {{ range .SchedulerArgs }} --kube-scheduler-arg={{ . }}{{- end }} --cluster-cidr={{ .PodSubnet }} --service-cidr={{ .ServiceSubnet }} --cluster-dns={{ .ClusterDns }} --flannel-backend=none --disable-network-policy --disable-cloud-controller --disable=servicelb,traefik,metrics-server,local-storage" -{{ end }} -Environment="K3S_EXTRA_ARGS={{ if .IsDockerRuntime }}--docker{{ end }} {{ if .ContainerRuntimeEndpoint }}--container-runtime-endpoint={{ .ContainerRuntimeEndpoint }}{{ end }} --node-name={{ .HostName }} --node-ip={{ .NodeIP }} --pause-image={{ .PauseImage }} {{ range .KubeletArgs }} --kubelet-arg={{ . }}{{- end }} {{ range .KubeProxyArgs }} --kube-proxy-arg={{ . }}{{- end }}" -Environment="K3S_ROLE={{ if .IsMaster }}server{{ else }}agent{{ end }}" -Environment="K3S_SERVER_ARGS={{ if .Server }}--server={{ .Server }}{{ end }}" -KillMode=process -Delegate=yes -LimitNOFILE=1048576 -LimitNPROC=infinity -LimitCORE=infinity -TasksMax=infinity -TimeoutStartSec=0 -Restart=always -RestartSec=5s -ExecStartPre=-/sbin/modprobe br_netfilter -ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k3s $K3S_ROLE $K3S_ARGS $K3S_EXTRA_ARGS $K3S_SERVER_ARGS - `))) -) diff --git a/cmd/kk/pkg/k3s/templates/k3sServiceEnv.go b/cmd/kk/pkg/k3s/templates/k3sServiceEnv.go deleted file mode 100644 index 49e92614..00000000 --- a/cmd/kk/pkg/k3s/templates/k3sServiceEnv.go +++ /dev/null @@ -1,44 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -// K3sServiceEnv defines the template of kubelet's Env for the kubelet's systemd service. -var K3sServiceEnv = template.Must(template.New("k3s.service.env").Parse( - dedent.Dedent(`# Note: This dropin only works with k3s -{{ if .IsMaster }} -K3S_DATASTORE_ENDPOINT={{ .DataStoreEndPoint }} -{{- if .DataStoreCaFile }} -K3S_DATASTORE_CAFILE={{ .DataStoreCaFile }} -{{- end }} -{{- if .DataStoreCertFile }} -K3S_DATASTORE_CERTFILE={{ .DataStoreCertFile }} -{{- end }} -{{- if .DataStoreKeyFile }} -K3S_DATASTORE_KEYFILE={{ .DataStoreKeyFile }} -{{- end }} -K3S_KUBECONFIG_MODE=644 -{{ end }} -{{ if .Token }} -K3S_TOKEN={{ .Token }} -{{ end }} - - `))) diff --git a/cmd/kk/pkg/k3s/templates/killall_script.go b/cmd/kk/pkg/k3s/templates/killall_script.go deleted file mode 100644 index 12eb709a..00000000 --- a/cmd/kk/pkg/k3s/templates/killall_script.go +++ /dev/null @@ -1,109 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -// K3sKillallScript defines the template of k3s-killall script. -var K3sKillallScript = template.Must(template.New("k3s-killall.sh").Parse( - dedent.Dedent(`#!/bin/sh -[ $(id -u) -eq 0 ] || exec sudo $0 $@ - -for bin in /var/lib/rancher/k3s/data/**/bin/; do - [ -d $bin ] && export PATH=$PATH:$bin:$bin/aux -done - -set -x - -for service in /etc/systemd/system/k3s*.service; do - [ -s $service ] && systemctl stop $(basename $service) -done - -for service in /etc/init.d/k3s*; do - [ -x $service ] && $service stop -done - -pschildren() { - ps -e -o ppid= -o pid= | \ - sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \ - grep -w "^$1" | \ - cut -f2 -} - -pstree() { - for pid in $@; do - echo $pid - for child in $(pschildren $pid); do - pstree $child - done - done -} - -killtree() { - kill -9 $( - { set +x; } 2>/dev/null; - pstree $@; - set -x; - ) 2>/dev/null -} - -getshims() { - ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1 -} - -killtree $({ set +x; } 2>/dev/null; getshims; set -x) - -do_unmount_and_remove() { - set +x - while read -r _ path _; do - case "$path" in $1*) echo "$path" ;; esac - done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount "$0" && rm -rf "$0"' - set -x -} - -do_unmount_and_remove '/run/k3s' -do_unmount_and_remove '/var/lib/rancher/k3s' -do_unmount_and_remove '/var/lib/kubelet/pods' -do_unmount_and_remove '/var/lib/kubelet/plugins' -do_unmount_and_remove '/run/netns/cni-' - -# Remove CNI namespaces -ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete - -# Delete network interface(s) that match 'master cni0' -ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do - iface=${iface%%@*} - [ -z "$iface" ] || ip link delete $iface -done -ip link delete cni0 -ip link delete flannel.1 -ip link delete flannel-v6.1 -ip link delete kube-ipvs0 -ip link delete flannel-wg -ip link delete flannel-wg-v6 -ip link delete nodelocaldns -ip link delete cilium_host -ip link delete cilium_vxlan -ip -br link show | grep 'cali[a-f0-9]*' | awk -F '@' '{print $1}' | xargs -r -t -n 1 ip link delete -rm -rf /var/lib/cni/ -iptables-save | grep -v KUBE- | grep -v CNI- | grep -iv flannel | iptables-restore -ip6tables-save | grep -v KUBE- | grep -v CNI- | grep -iv flannel | ip6tables-restore - `))) diff --git a/cmd/kk/pkg/k3s/templates/uninstall_script.go b/cmd/kk/pkg/k3s/templates/uninstall_script.go deleted file mode 100644 index 797b8672..00000000 --- a/cmd/kk/pkg/k3s/templates/uninstall_script.go +++ /dev/null @@ -1,73 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -// K3sUninstallScript defines the template of k3s-killall script. -var K3sUninstallScript = template.Must(template.New("k3s-uninstall.sh").Parse( - dedent.Dedent(`#!/bin/sh -set -x -[ $(id -u) -eq 0 ] || exec sudo $0 $@ - -/usr/local/bin/k3s-killall.sh - -if which systemctl; then - systemctl disable k3s - systemctl reset-failed k3s - systemctl daemon-reload -fi -if which rc-update; then - rc-update delete k3s default -fi - -rm -f /etc/systemd/system/k3s.service -rm -rf /etc/systemd/system/k3s.service.d -rm -f /etc/systemd/system/k3s.service.env - -remove_uninstall() { - rm -f /usr/local/bin/k3s-uninstall.sh -} -trap remove_uninstall EXIT - -if (ls /etc/systemd/system/k3s*.service || ls /etc/init.d/k3s*) >/dev/null 2>&1; then - set +x; echo 'Additional k3s services installed, skipping uninstall of k3s'; set -x - exit -fi - -for cmd in kubectl crictl ctr; do - if [ -L /usr/local/bin/$cmd ]; then - rm -f /usr/local/bin/$cmd - fi -done - -rm -rf /etc/rancher/k3s -rm -rf /run/k3s -rm -rf /run/flannel -rm -rf /var/lib/rancher/k3s -rm -rf /var/lib/kubelet -rm -f /usr/local/bin/k3s -rm -f /usr/local/bin/k3s-killall.sh - -if type yum >/dev/null 2>&1; then - yum remove -y k3s-selinux - rm -f /etc/yum.repos.d/rancher-k3s-common*.repo -fi - `))) diff --git a/cmd/kk/pkg/k8e/k8e_status.go b/cmd/kk/pkg/k8e/k8e_status.go deleted file mode 100644 index 5315d063..00000000 --- a/cmd/kk/pkg/k8e/k8e_status.go +++ /dev/null @@ -1,126 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k8e - -import ( - "fmt" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type K8eStatus struct { - Version string - ClusterInfo string - NodeToken string - KubeConfig string - NodesInfo map[string]string -} - -func NewK8eStatus() *K8eStatus { - return &K8eStatus{NodesInfo: make(map[string]string)} -} - -func (k *K8eStatus) SearchVersion(runtime connector.Runtime) error { - cmd := "/usr/local/bin/k8e --version | grep 'k8e' | awk '{print $3}'" - if output, err := runtime.GetRunner().Cmd(cmd, true); err != nil { - return errors.Wrap(errors.WithStack(err), "search current version failed") - } else { - k.Version = output - } - return nil -} - -func (k *K8eStatus) SearchKubeConfig(runtime connector.Runtime) error { - kubeCfgCmd := "cat /etc/k8e/k8e.yaml" - if kubeConfigStr, err := runtime.GetRunner().SudoCmd(kubeCfgCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "search cluster kubeconfig failed") - } else { - k.KubeConfig = kubeConfigStr - } - return nil -} - -func (k *K8eStatus) SearchNodeToken(runtime connector.Runtime) error { - nodeTokenBase64Cmd := "cat /var/lib/k8e/server/node-token" - output, err := runtime.GetRunner().SudoCmd(nodeTokenBase64Cmd, true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get cluster node token failed") - } - k.NodeToken = output - return nil -} - -func (k *K8eStatus) SearchInfo(runtime connector.Runtime) error { - output, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl --no-headers=true get nodes -o custom-columns=:metadata.name,:status.nodeInfo.kubeletVersion,:status.addresses", - true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get K8e cluster info failed") - } - k.ClusterInfo = output - return nil -} - -func (k *K8eStatus) SearchNodesInfo(_ connector.Runtime) error { - ipv4Regexp, err := regexp.Compile(common.IPv4Regexp) - if err != nil { - return err - } - ipv6Regexp, err := regexp.Compile(common.IPv6Regexp) - if err != nil { - return err - } - tmp := strings.Split(k.ClusterInfo, "\r\n") - if len(tmp) < 1 || len(tmp) == 1 && tmp[0] == "" { - return errors.New("search K8e node info failed") - } - - for i := 0; i < len(tmp); i++ { - if ipv4 := ipv4Regexp.FindStringSubmatch(tmp[i]); len(ipv4) != 0 { - k.NodesInfo[ipv4[0]] = ipv4[0] - } - if ipv6 := ipv6Regexp.FindStringSubmatch(tmp[i]); len(ipv6) != 0 { - k.NodesInfo[ipv6[0]] = ipv6[0] - } - if len(strings.Fields(tmp[i])) > 3 { - k.NodesInfo[strings.Fields(tmp[i])[0]] = strings.Fields(tmp[i])[1] - } else { - k.NodesInfo[strings.Fields(tmp[i])[0]] = "" - } - } - return nil -} - -func (k *K8eStatus) LoadKubeConfig(runtime connector.Runtime, kubeConf *common.KubeConf) error { - kubeConfigPath := filepath.Join(runtime.GetWorkDir(), fmt.Sprintf("config-%s", runtime.GetObjName())) - - oldServer := "server: https://127.0.0.1:6443" - newServer := fmt.Sprintf("server: https://%s:%d", kubeConf.Cluster.ControlPlaneEndpoint.Address, kubeConf.Cluster.ControlPlaneEndpoint.Port) - newKubeConfigStr := strings.Replace(k.KubeConfig, oldServer, newServer, -1) - - if err := os.WriteFile(kubeConfigPath, []byte(newKubeConfigStr), 0644); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/k8e/module.go b/cmd/kk/pkg/k8e/module.go deleted file mode 100644 index d5184852..00000000 --- a/cmd/kk/pkg/k8e/module.go +++ /dev/null @@ -1,316 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k8e - -import ( - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k8e/templates" -) - -type StatusModule struct { - common.KubeModule -} - -func (s *StatusModule) Init() { - s.Name = "StatusModule" - s.Desc = "Get cluster status" - - cluster := NewK8eStatus() - s.PipelineCache.GetOrSet(common.ClusterStatus, cluster) - - clusterStatus := &task.RemoteTask{ - Name: "GetClusterStatus", - Desc: "Get K8e cluster status", - Hosts: s.Runtime.GetHostsByRole(common.Master), - Action: new(GetClusterStatus), - Parallel: false, - } - - s.Tasks = []task.Interface{ - clusterStatus, - } -} - -type InstallKubeBinariesModule struct { - common.KubeModule -} - -func (i *InstallKubeBinariesModule) Init() { - i.Name = "InstallKubeBinariesModule" - i.Desc = "Install k8e cluster" - - syncBinary := &task.RemoteTask{ - Name: "SyncKubeBinary", - Desc: "Synchronize k8e binaries", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: new(SyncKubeBinary), - Parallel: true, - Retry: 2, - } - - killAllScript := &task.RemoteTask{ - Name: "GenerateK8eKillAllScript", - Desc: "Generate k8e killall.sh script", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: &action.Template{ - Template: templates.K8eKillallScript, - Dst: filepath.Join("/usr/local/bin", templates.K8eKillallScript.Name()), - }, - Parallel: true, - Retry: 2, - } - - uninstallScript := &task.RemoteTask{ - Name: "GenerateK8eUninstallScript", - Desc: "Generate k8e uninstall script", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: &action.Template{ - Template: templates.K8eUninstallScript, - Dst: filepath.Join("/usr/local/bin", templates.K8eUninstallScript.Name()), - }, - Parallel: true, - Retry: 2, - } - - chmod := &task.RemoteTask{ - Name: "ChmodScript", - Desc: "Chmod +x k8e script ", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: new(ChmodScript), - Parallel: true, - Retry: 2, - } - - i.Tasks = []task.Interface{ - syncBinary, - killAllScript, - uninstallScript, - chmod, - } -} - -type InitClusterModule struct { - common.KubeModule -} - -func (i *InitClusterModule) Init() { - i.Name = "K8eInitClusterModule" - i.Desc = "Init k8e cluster" - - k8eService := &task.RemoteTask{ - Name: "GenerateK8eService", - Desc: "Generate k8e Service", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(GenerateK8eService), - Parallel: true, - } - - k8eEnv := &task.RemoteTask{ - Name: "GenerateK8eServiceEnv", - Desc: "Generate k8e service env", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(GenerateK8eServiceEnv), - Parallel: true, - } - - enableK8e := &task.RemoteTask{ - Name: "EnableK8eService", - Desc: "Enable k8e service", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(EnableK8eService), - Parallel: true, - } - - copyKubeConfig := &task.RemoteTask{ - Name: "CopyKubeConfig", - Desc: "Copy k8e.yaml to ~/.kube/config", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(CopyK8eKubeConfig), - Parallel: true, - } - - addMasterTaint := &task.RemoteTask{ - Name: "AddMasterTaint", - Desc: "Add master taint", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - &common.IsWorker{Not: true}, - }, - Action: new(AddMasterTaint), - Parallel: true, - Retry: 5, - } - - i.Tasks = []task.Interface{ - k8eService, - k8eEnv, - enableK8e, - copyKubeConfig, - addMasterTaint, - } -} - -type JoinNodesModule struct { - common.KubeModule -} - -func (j *JoinNodesModule) Init() { - j.Name = "K8eJoinNodesModule" - j.Desc = "Join k8e nodes" - - k8eService := &task.RemoteTask{ - Name: "GenerateK8eService", - Desc: "Generate k8e Service", - Hosts: j.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(GenerateK8eService), - Parallel: true, - } - - k8eEnv := &task.RemoteTask{ - Name: "GenerateK8eServiceEnv", - Desc: "Generate k8e service env", - Hosts: j.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(GenerateK8eServiceEnv), - Parallel: true, - } - - enableK8e := &task.RemoteTask{ - Name: "EnableK8eService", - Desc: "Enable k8e service", - Hosts: j.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(EnableK8eService), - Parallel: true, - } - - copyKubeConfigForMaster := &task.RemoteTask{ - Name: "CopyKubeConfig", - Desc: "Copy k8e.yaml to ~/.kube/config", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(CopyK8eKubeConfig), - Parallel: true, - } - - addMasterTaint := &task.RemoteTask{ - Name: "AddMasterTaint", - Desc: "Add master taint", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - &common.IsWorker{Not: true}, - }, - Action: new(AddMasterTaint), - Parallel: true, - Retry: 5, - } - - addWorkerLabelToNode := &task.RemoteTask{ - Name: "addWorkerLabelToNode", - Desc: "Add worker label to all nodes", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(AddWorkerLabel), - Retry: 3, - } - - j.Tasks = []task.Interface{ - k8eService, - k8eEnv, - enableK8e, - copyKubeConfigForMaster, - addMasterTaint, - addWorkerLabelToNode, - } -} - -type DeleteClusterModule struct { - common.KubeModule -} - -func (d *DeleteClusterModule) Init() { - d.Name = "DeleteClusterModule" - d.Desc = "Delete k8e cluster" - - execScript := &task.RemoteTask{ - Name: "ExecUninstallScript", - Desc: "Exec k8e uninstall script", - Hosts: d.Runtime.GetHostsByRole(common.K8s), - Action: new(ExecUninstallScript), - Parallel: true, - } - - d.Tasks = []task.Interface{ - execScript, - } -} - -type SaveKubeConfigModule struct { - common.KubeModule -} - -func (s *SaveKubeConfigModule) Init() { - s.Name = "SaveKubeConfigModule" - s.Desc = "Save kube config file as a configmap" - - save := &task.LocalTask{ - Name: "SaveKubeConfig", - Desc: "Save kube config as a configmap", - Action: new(SaveKubeConfig), - } - - s.Tasks = []task.Interface{ - save, - } -} diff --git a/cmd/kk/pkg/k8e/prepares.go b/cmd/kk/pkg/k8e/prepares.go deleted file mode 100644 index 7a1d100a..00000000 --- a/cmd/kk/pkg/k8e/prepares.go +++ /dev/null @@ -1,62 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k8e - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type NodeInCluster struct { - common.KubePrepare - Not bool -} - -func (n *NodeInCluster) PreCheck(runtime connector.Runtime) (bool, error) { - host := runtime.RemoteHost() - if v, ok := n.PipelineCache.Get(common.ClusterStatus); ok { - cluster := v.(*K8eStatus) - var versionOk bool - if res, ok := cluster.NodesInfo[host.GetName()]; ok && res != "" { - versionOk = true - } - _, ipOk := cluster.NodesInfo[host.GetInternalAddress()] - if n.Not { - return !(versionOk || ipOk), nil - } - return versionOk || ipOk, nil - } - return false, errors.New("get k8e cluster status by pipeline cache failed") -} - -type ClusterIsExist struct { - common.KubePrepare - Not bool -} - -func (c *ClusterIsExist) PreCheck(_ connector.Runtime) (bool, error) { - if exist, ok := c.PipelineCache.GetMustBool(common.ClusterExist); ok { - if c.Not { - return !exist, nil - } - return exist, nil - } else { - return false, errors.New("get k8e cluster status by pipeline cache failed") - } -} diff --git a/cmd/kk/pkg/k8e/tasks.go b/cmd/kk/pkg/k8e/tasks.go deleted file mode 100644 index c1064dc4..00000000 --- a/cmd/kk/pkg/k8e/tasks.go +++ /dev/null @@ -1,458 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k8e - -import ( - "context" - "encoding/base64" - "fmt" - "path/filepath" - "strings" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - versionutil "k8s.io/apimachinery/pkg/util/version" - kube "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k8e/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -type GetClusterStatus struct { - common.KubeAction -} - -func (g *GetClusterStatus) Execute(runtime connector.Runtime) error { - exist, err := runtime.GetRunner().FileExist("/etc/systemd/system/k8e.service") - if err != nil { - return err - } - - if !exist { - g.PipelineCache.Set(common.ClusterExist, false) - return nil - } else { - g.PipelineCache.Set(common.ClusterExist, true) - - if v, ok := g.PipelineCache.Get(common.ClusterStatus); ok { - cluster := v.(*K8eStatus) - if err := cluster.SearchVersion(runtime); err != nil { - return err - } - if err := cluster.SearchKubeConfig(runtime); err != nil { - return err - } - if err := cluster.LoadKubeConfig(runtime, g.KubeConf); err != nil { - return err - } - if err := cluster.SearchNodeToken(runtime); err != nil { - return err - } - if err := cluster.SearchInfo(runtime); err != nil { - return err - } - if err := cluster.SearchNodesInfo(runtime); err != nil { - return err - } - g.PipelineCache.Set(common.ClusterStatus, cluster) - } else { - return errors.New("get k8e cluster status by pipeline cache failed") - } - } - return nil -} - -type SyncKubeBinary struct { - common.KubeAction -} - -func (s *SyncKubeBinary) Execute(runtime connector.Runtime) error { - binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - if err := SyncKubeBinaries(runtime, binariesMap); err != nil { - return err - } - return nil -} - -// SyncKubeBinaries is used to sync kubernetes' binaries to each node. -func SyncKubeBinaries(runtime connector.Runtime, binariesMap map[string]*files.KubeBinary) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binaryList := []string{"k8e", "helm", "kubecni"} - for _, name := range binaryList { - binary, ok := binariesMap[name] - if !ok { - return fmt.Errorf("get kube binary %s info failed: no such key", name) - } - - fileName := binary.FileName - switch name { - case "kubecni": - dst := filepath.Join(common.TmpDir, fileName) - if err := runtime.GetRunner().Scp(binary.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed")) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("tar -zxf %s -C /opt/cni/bin", dst), false); err != nil { - return err - } - default: - dst := filepath.Join(common.BinDir, fileName) - if err := runtime.GetRunner().SudoScp(binary.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed")) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s", dst), false); err != nil { - return err - } - } - } - - binaries := []string{"kubectl", "crictl", "ctr"} - var createLinkCMDs []string - for _, binary := range binaries { - createLinkCMDs = append(createLinkCMDs, fmt.Sprintf("ln -snf /usr/local/bin/k8e /usr/local/bin/%s", binary)) - } - if _, err := runtime.GetRunner().SudoCmd(strings.Join(createLinkCMDs, " && "), false); err != nil { - return errors.Wrap(errors.WithStack(err), "create ctl tool link failed") - } - - return nil -} - -type ChmodScript struct { - common.KubeAction -} - -func (c *ChmodScript) Execute(runtime connector.Runtime) error { - killAllScript := filepath.Join("/usr/local/bin", templates.K8eKillallScript.Name()) - uninstallScript := filepath.Join("/usr/local/bin", templates.K8eUninstallScript.Name()) - - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s", killAllScript), - false); err != nil { - return err - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s", uninstallScript), - false); err != nil { - return err - } - return nil -} - -type GenerateK8eService struct { - common.KubeAction -} - -func (g *GenerateK8eService) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - var server string - if !host.IsRole(common.Master) { - server = fmt.Sprintf("https://%s:%d", g.KubeConf.Cluster.ControlPlaneEndpoint.Domain, g.KubeConf.Cluster.ControlPlaneEndpoint.Port) - } - - defaultKubeletArs := map[string]string{ - "kube-reserved": "cpu=200m,memory=250Mi,ephemeral-storage=1Gi", - "system-reserved": "cpu=200m,memory=250Mi,ephemeral-storage=1Gi", - "eviction-hard": "memory.available<5%,nodefs.available<10%", - } - defaultKubeProxyArgs := map[string]string{ - "proxy-mode": "ipvs", - } - - kubeApiserverArgs, _ := util.GetArgs(map[string]string{}, g.KubeConf.Cluster.Kubernetes.ApiServerArgs) - kubeControllerManager, _ := util.GetArgs(map[string]string{ - "pod-eviction-timeout": "3m0s", - "terminated-pod-gc-threshold": "5", - }, g.KubeConf.Cluster.Kubernetes.ControllerManagerArgs) - kubeSchedulerArgs, _ := util.GetArgs(map[string]string{}, g.KubeConf.Cluster.Kubernetes.SchedulerArgs) - kubeletArgs, _ := util.GetArgs(defaultKubeletArs, g.KubeConf.Cluster.Kubernetes.KubeletArgs) - kubeProxyArgs, _ := util.GetArgs(defaultKubeProxyArgs, g.KubeConf.Cluster.Kubernetes.KubeProxyArgs) - - templateAction := action.Template{ - Template: templates.K8eService, - Dst: filepath.Join("/etc/systemd/system/", templates.K8eService.Name()), - Data: util.Data{ - "Server": server, - "IsMaster": host.IsRole(common.Master), - "NodeIP": host.GetInternalAddress(), - "HostName": host.GetName(), - "PodSubnet": g.KubeConf.Cluster.Network.KubePodsCIDR, - "ServiceSubnet": g.KubeConf.Cluster.Network.KubeServiceCIDR, - "ClusterDns": g.KubeConf.Cluster.CorednsClusterIP(), - "CertSANs": g.KubeConf.Cluster.GenerateCertSANs(), - "PauseImage": images.GetImage(runtime, g.KubeConf, "pause").ImageName(), - "ApiserverArgs": kubeApiserverArgs, - "ControllerManager": kubeControllerManager, - "SchedulerArgs": kubeSchedulerArgs, - "KubeletArgs": kubeletArgs, - "KubeProxyArgs": kubeProxyArgs, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type GenerateK8eServiceEnv struct { - common.KubeAction -} - -func (g *GenerateK8eServiceEnv) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - clusterStatus, ok := g.PipelineCache.Get(common.ClusterStatus) - if !ok { - return errors.New("get cluster status by pipeline cache failed") - } - cluster := clusterStatus.(*K8eStatus) - - var externalEtcd kubekeyapiv1alpha2.ExternalEtcd - var endpointsList []string - var externalEtcdEndpoints, token string - - switch g.KubeConf.Cluster.Etcd.Type { - case kubekeyapiv1alpha2.External: - externalEtcd.Endpoints = g.KubeConf.Cluster.Etcd.External.Endpoints - - if len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 { - externalEtcd.CAFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CAFile)) - externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CertFile)) - externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.KeyFile)) - } - default: - for _, node := range runtime.GetHostsByRole(common.ETCD) { - endpoint := fmt.Sprintf("https://%s:%s", node.GetInternalAddress(), kubekeyapiv1alpha2.DefaultEtcdPort) - endpointsList = append(endpointsList, endpoint) - } - externalEtcd.Endpoints = endpointsList - - externalEtcd.CAFile = "/etc/ssl/etcd/ssl/ca.pem" - externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s.pem", runtime.GetHostsByRole(common.Master)[0].GetName()) - externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s-key.pem", runtime.GetHostsByRole(common.Master)[0].GetName()) - } - - externalEtcdEndpoints = strings.Join(externalEtcd.Endpoints, ",") - - v121 := versionutil.MustParseSemantic("v1.21.0") - atLeast := versionutil.MustParseSemantic(g.KubeConf.Cluster.Kubernetes.Version).AtLeast(v121) - if atLeast { - token = cluster.NodeToken - } else { - if !host.IsRole(common.Master) { - token = cluster.NodeToken - } - } - - templateAction := action.Template{ - Template: templates.K8eServiceEnv, - Dst: filepath.Join("/etc/systemd/system/", templates.K8eServiceEnv.Name()), - Data: util.Data{ - "DataStoreEndPoint": externalEtcdEndpoints, - "DataStoreCaFile": externalEtcd.CAFile, - "DataStoreCertFile": externalEtcd.CertFile, - "DataStoreKeyFile": externalEtcd.KeyFile, - "IsMaster": host.IsRole(common.Master), - "Token": token, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type EnableK8eService struct { - common.KubeAction -} - -func (e *EnableK8eService) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && systemctl enable --now k8e", - false); err != nil { - return errors.Wrap(errors.WithStack(err), "enable k8e failed") - } - return nil -} - -type CopyK8eKubeConfig struct { - common.KubeAction -} - -func (c *CopyK8eKubeConfig) Execute(runtime connector.Runtime) error { - createConfigDirCmd := "mkdir -p /root/.kube && mkdir -p $HOME/.kube" - getKubeConfigCmd := "cp -f /etc/k8e/k8e.yaml /root/.kube/config" - - cmd := strings.Join([]string{createConfigDirCmd, getKubeConfigCmd}, " && ") - if _, err := runtime.GetRunner().SudoCmd(cmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "copy k8e kube config failed") - } - - userMkdir := "mkdir -p $HOME/.kube" - if _, err := runtime.GetRunner().Cmd(userMkdir, false); err != nil { - return errors.Wrap(errors.WithStack(err), "user mkdir $HOME/.kube failed") - } - - userCopyKubeConfig := "cp -f /etc/k8e/k8e.yaml $HOME/.kube/config" - if _, err := runtime.GetRunner().SudoCmd(userCopyKubeConfig, false); err != nil { - return errors.Wrap(errors.WithStack(err), "user copy /etc/k8e/k8e.yaml to $HOME/.kube/config failed") - } - - userId, err := runtime.GetRunner().Cmd("echo $(id -u)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user id failed") - } - - userGroupId, err := runtime.GetRunner().Cmd("echo $(id -g)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user group id failed") - } - - chownKubeConfig := fmt.Sprintf("chown -R %s:%s $HOME/.kube", userId, userGroupId) - if _, err := runtime.GetRunner().SudoCmd(chownKubeConfig, false); err != nil { - return errors.Wrap(errors.WithStack(err), "chown user kube config failed") - } - return nil -} - -type AddMasterTaint struct { - common.KubeAction -} - -func (a *AddMasterTaint) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - cmd := fmt.Sprintf( - "/usr/local/bin/kubectl taint nodes %s node-role.kubernetes.io/master=effect:NoSchedule --overwrite", - host.GetName()) - - if _, err := runtime.GetRunner().SudoCmd(cmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "add master NoSchedule taint failed") - } - return nil -} - -type AddWorkerLabel struct { - common.KubeAction -} - -func (a *AddWorkerLabel) Execute(runtime connector.Runtime) error { - for _, host := range runtime.GetAllHosts() { - if host.IsRole(common.Worker) { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "/usr/local/bin/kubectl label --overwrite node %s node-role.kubernetes.io/worker=", - host.GetName()), true); err != nil { - return errors.Wrap(errors.WithStack(err), "add worker label failed") - } - } - } - - return nil -} - -type ExecUninstallScript struct { - common.KubeAction -} - -func (e *ExecUninstallScript) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && /usr/local/bin/k8e-killall.sh", - true); err != nil { - return errors.Wrap(errors.WithStack(err), "add master NoSchedule taint failed") - } - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && /usr/local/bin/k8e-uninstall.sh", - true); err != nil { - return errors.Wrap(errors.WithStack(err), "add master NoSchedule taint failed") - } - return nil -} - -type SaveKubeConfig struct { - common.KubeAction -} - -func (s *SaveKubeConfig) Execute(_ connector.Runtime) error { - status, ok := s.PipelineCache.Get(common.ClusterStatus) - if !ok { - return errors.New("get kubernetes status failed by pipeline cache") - } - cluster := status.(*K8eStatus) - - oldServer := fmt.Sprintf("https://%s:%d", s.KubeConf.Cluster.ControlPlaneEndpoint.Domain, s.KubeConf.Cluster.ControlPlaneEndpoint.Port) - newServer := fmt.Sprintf("https://%s:%d", s.KubeConf.Cluster.ControlPlaneEndpoint.Address, s.KubeConf.Cluster.ControlPlaneEndpoint.Port) - newKubeConfigStr := strings.Replace(cluster.KubeConfig, oldServer, newServer, -1) - kubeConfigBase64 := base64.StdEncoding.EncodeToString([]byte(newKubeConfigStr)) - - config, err := clientcmd.NewClientConfigFromBytes([]byte(newKubeConfigStr)) - if err != nil { - return err - } - restConfig, err := config.ClientConfig() - if err != nil { - return err - } - clientsetForCluster, err := kube.NewForConfig(restConfig) - if err != nil { - return err - } - - namespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubekey-system", - }, - } - if _, err := clientsetForCluster. - CoreV1(). - Namespaces(). - Create(context.TODO(), namespace, metav1.CreateOptions{}); err != nil { - return err - } - - cm := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-kubeconfig", s.KubeConf.ClusterName), - }, - Data: map[string]string{ - "kubeconfig": kubeConfigBase64, - }, - } - - if _, err := clientsetForCluster. - CoreV1(). - ConfigMaps("kubekey-system"). - Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/k8e/templates/k8eService.go b/cmd/kk/pkg/k8e/templates/k8eService.go deleted file mode 100644 index 3f974bd9..00000000 --- a/cmd/kk/pkg/k8e/templates/k8eService.go +++ /dev/null @@ -1,59 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var ( - // K8eService defines the template of kubelet service for systemd. - K8eService = template.Must(template.New("k8e.service").Parse( - dedent.Dedent(`[Unit] -Description=Simple Kubernetes Distribution -Documentation=https://getk8e.com -Wants=network-online.target -After=network-online.target - -[Install] -WantedBy=multi-user.target - -[Service] -Type=notify -EnvironmentFile=/etc/systemd/system/k8e.service.env -{{ if .IsMaster }} -Environment="K8E_ARGS= {{ range .CertSANs }} --tls-san={{ . }}{{- end }} {{ range .ApiserverArgs }} --kube-apiserver-arg={{ . }}{{- end }} {{ range .ControllerManager }} --kube-controller-manager-arg={{ . }}{{- end }} {{ range .SchedulerArgs }} --kube-scheduler-arg={{ . }}{{- end }} --cluster-cidr={{ .PodSubnet }} --service-cidr={{ .ServiceSubnet }} --cluster-dns={{ .ClusterDns }} " -{{ end }} -Environment="K8E_EXTRA_ARGS=--node-name={{ .HostName }} --node-ip={{ .NodeIP }} --pause-image={{ .PauseImage }} {{ range .KubeletArgs }} --kubelet-arg={{ . }}{{- end }} {{ range .KubeProxyArgs }} --kube-proxy-arg={{ . }}{{- end }}" -Environment="K8E_ROLE={{ if .IsMaster }}server{{ else }}agent{{ end }}" -Environment="K8E_SERVER_ARGS={{ if .Server }}--server={{ .Server }}{{ end }}" -KillMode=process -Delegate=yes -LimitNOFILE=1048576 -LimitNPROC=infinity -LimitCORE=infinity -TasksMax=infinity -TimeoutStartSec=0 -Restart=always -RestartSec=5s -ExecStartPre=-/sbin/modprobe br_netfilter -ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k8e $K8E_ROLE $K8E_ARGS $K8E_EXTRA_ARGS $K8E_SERVER_ARGS - `))) -) diff --git a/cmd/kk/pkg/k8e/templates/k8eServiceEnv.go b/cmd/kk/pkg/k8e/templates/k8eServiceEnv.go deleted file mode 100644 index e4985c28..00000000 --- a/cmd/kk/pkg/k8e/templates/k8eServiceEnv.go +++ /dev/null @@ -1,45 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -// K8eServiceEnv defines the template of kubelet's Env for the kubelet's systemd service. -var K8eServiceEnv = template.Must(template.New("k8e.service.env").Parse( - dedent.Dedent(`# Note: This dropin only works with k3s -{{ if .IsMaster }} -K8E_DATASTORE_ENDPOINT={{ .DataStoreEndPoint }} -{{- if .DataStoreCaFile }} -K8E_DATASTORE_CAFILE={{ .DataStoreCaFile }} -{{- end }} -{{- if .DataStoreCertFile }} -K8E_DATASTORE_CERTFILE={{ .DataStoreCertFile }} -{{- end }} -{{- if .DataStoreKeyFile }} -K8E_DATASTORE_KEYFILE={{ .DataStoreKeyFile }} -{{- end }} -K8E_KUBECONFIG_MODE=644 -{{ end }} -{{ if .Token }} -K8E_TOKEN={{ .Token }} -{{ end }} - - `))) diff --git a/cmd/kk/pkg/k8e/templates/killall_script.go b/cmd/kk/pkg/k8e/templates/killall_script.go deleted file mode 100644 index 9ae34db7..00000000 --- a/cmd/kk/pkg/k8e/templates/killall_script.go +++ /dev/null @@ -1,73 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -// K8eKillallScript defines the template of k3s-killall script. -var K8eKillallScript = template.Must(template.New("k8e-killall.sh").Parse( - dedent.Dedent(`#!/bin/sh -[ $(id -u) -eq 0 ] || exec sudo $0 $@ - -for service in /etc/systemd/system/k8e*.service; do - [ -s $service ] && systemctl stop $(basename $service) -done -pschildren() { - ps -e -o ppid= -o pid= | \ - sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \ - grep -w "^$1" | \ - cut -f2 -} -pstree() { - for pid in $@; do - echo $pid - for child in $(pschildren $pid); do - pstree $child - done - done -} -killtree() { - kill -9 $( - { set +x; } 2>/dev/null; - pstree $@; - set -x; - ) 2>/dev/null -} -getshims() { - ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k8e/data/[^/]*/bin/containerd-shim' | cut -f1 -} -killtree $({ set +x; } 2>/dev/null; getshims; set -x) -do_unmount_and_remove() { - set +x - while read -r _ path _; do - case "$path" in $1*) echo "$path" ;; esac - done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount "$0" && rm -rf "$0"' - set -x -} -do_unmount_and_remove '/run/k8e' -do_unmount_and_remove '/var/lib/k8e' -do_unmount_and_remove '/var/lib/kubelet/pods' -do_unmount_and_remove '/var/lib/kubelet/plugins' -do_unmount_and_remove '/run/netns/cni-' -# Remove CNI namespaces -ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete -rm -rf /var/lib/cni/ - `))) diff --git a/cmd/kk/pkg/k8e/templates/uninstall_script.go b/cmd/kk/pkg/k8e/templates/uninstall_script.go deleted file mode 100644 index dc793a99..00000000 --- a/cmd/kk/pkg/k8e/templates/uninstall_script.go +++ /dev/null @@ -1,68 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -// K8eUninstallScript defines the template of k3s-killall script. -var K8eUninstallScript = template.Must(template.New("k8e-uninstall.sh").Parse( - dedent.Dedent(`#!/bin/sh -set -x -[ $(id -u) -eq 0 ] || exec sudo $0 $@ - -/usr/local/bin/k8e-killall.sh - -if which systemctl; then - systemctl disable k8e - systemctl reset-failed k8e - systemctl daemon-reload -fi -if which rc-update; then - rc-update delete k8e default -fi - -rm -f /etc/systemd/system/k8e.service -rm -rf /etc/systemd/system/k8e.service.d -rm -f /etc/systemd/system/k8e.service.env - -remove_uninstall() { - rm -f /usr/local/bin/k8e-uninstall.sh -} -trap remove_uninstall EXIT - -if (ls /etc/systemd/system/k8e*.service || ls /etc/init.d/k8e*) >/dev/null 2>&1; then - set +x; echo 'Additional k8e services installed, skipping uninstall of k8e'; set -x - exit -fi - -for cmd in kubectl crictl ctr; do - if [ -L /usr/local/bin/$cmd ]; then - rm -f /usr/local/bin/$cmd - fi -done - -rm -rf /etc/rancher/k8e -rm -rf /run/k8e -rm -rf /var/lib/k8e -rm -rf /var/lib/kubelet -rm -f /usr/local/bin/k8e -rm -f /usr/local/bin/k8e-killall.sh - `))) diff --git a/cmd/kk/pkg/kubernetes/kubernetes_status.go b/cmd/kk/pkg/kubernetes/kubernetes_status.go deleted file mode 100644 index 582d5f59..00000000 --- a/cmd/kk/pkg/kubernetes/kubernetes_status.go +++ /dev/null @@ -1,169 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "fmt" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type KubernetesStatus struct { - Version string - BootstrapToken string - CertificateKey string - ClusterInfo string - KubeConfig string - NodesInfo map[string]string -} - -func NewKubernetesStatus() *KubernetesStatus { - return &KubernetesStatus{NodesInfo: make(map[string]string)} -} - -func (k *KubernetesStatus) SearchVersion(runtime connector.Runtime) error { - cmd := "sudo cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep 'image:' | awk -F '[:]' '{print $(NF-0)}'" - if output, err := runtime.GetRunner().Cmd(cmd, true); err != nil { - return errors.Wrap(errors.WithStack(err), "search current version failed") - } else { - if !strings.Contains(output, "No such file or directory") { - k.Version = output - } - } - return nil -} - -func (k *KubernetesStatus) SearchJoinInfo(runtime connector.Runtime) error { - checkKubeadmConfig, err := runtime.GetRunner().SudoCmd("cat /etc/kubernetes/kubeadm-config.yaml", false) - if err != nil { - return err - } - if (k.BootstrapToken != "" || k.CertificateKey != "") && - (!strings.Contains(checkKubeadmConfig, "InitConfiguration") || - !strings.Contains(checkKubeadmConfig, "ClusterConfiguration")) { - return nil - } - - uploadCertsCmd := "/usr/local/bin/kubeadm init phase upload-certs --upload-certs --config /etc/kubernetes/kubeadm-config.yaml" - output, err := runtime.GetRunner().SudoCmd(uploadCertsCmd, true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "Failed to upload kubeadm certs") - } - reg := regexp.MustCompile("[0-9|a-z]{64}") - k.CertificateKey = reg.FindAllString(output, -1)[0] - - if err := patchKubeadmSecret(runtime); err != nil { - return err - } - - tokenCreateMasterCmd := "/usr/local/bin/kubeadm token create" - token, err := runtime.GetRunner().SudoCmd(tokenCreateMasterCmd, true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "Failed to get join node cmd") - } - - reg = regexp.MustCompile("[0-9|a-z]{6}.[0-9|a-z]{16}") - k.BootstrapToken = reg.FindAllString(token, -1)[0] - return nil -} - -func (k *KubernetesStatus) SearchClusterInfo(runtime connector.Runtime) error { - output, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl --no-headers=true get nodes -o custom-columns=:metadata.name,:status.nodeInfo.kubeletVersion,:status.addresses", - true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get kubernetes cluster info failed") - } - k.ClusterInfo = output - return nil -} - -func (k *KubernetesStatus) SearchNodesInfo(_ connector.Runtime) error { - ipv4Regexp, err := regexp.Compile(common.IPv4Regexp) - if err != nil { - return err - } - ipv6Regexp, err := regexp.Compile(common.IPv6Regexp) - if err != nil { - return err - } - tmp := strings.Split(k.ClusterInfo, "\r\n") - if len(tmp) >= 1 { - for i := 0; i < len(tmp); i++ { - if ipv4 := ipv4Regexp.FindStringSubmatch(tmp[i]); len(ipv4) != 0 { - k.NodesInfo[ipv4[0]] = ipv4[0] - } - if ipv6 := ipv6Regexp.FindStringSubmatch(tmp[i]); len(ipv6) != 0 { - k.NodesInfo[ipv6[0]] = ipv6[0] - } - if len(strings.Fields(tmp[i])) > 3 { - k.NodesInfo[strings.Fields(tmp[i])[0]] = strings.Fields(tmp[i])[1] - } else { - k.NodesInfo[strings.Fields(tmp[i])[0]] = "" - } - } - } - return nil -} - -func (k *KubernetesStatus) SearchKubeConfig(runtime connector.Runtime) error { - kubeCfgCmd := "cat /etc/kubernetes/admin.conf" - if kubeConfigStr, err := runtime.GetRunner().SudoCmd(kubeCfgCmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "search cluster kubeconfig failed") - } else { - k.KubeConfig = kubeConfigStr - } - return nil -} - -func (k *KubernetesStatus) LoadKubeConfig(runtime connector.Runtime, kubeConf *common.KubeConf) error { - kubeConfigPath := filepath.Join(runtime.GetWorkDir(), fmt.Sprintf("config-%s", runtime.GetObjName())) - kubeConfigStr := k.KubeConfig - - oldServer := fmt.Sprintf("server: https://%s:%d", kubeConf.Cluster.ControlPlaneEndpoint.Domain, kubeConf.Cluster.ControlPlaneEndpoint.Port) - if kubeConf.Cluster.ControlPlaneEndpoint.Address == "" { - kubeConf.Cluster.ControlPlaneEndpoint.Address = runtime.GetHostsByRole(common.Master)[0].GetAddress() - } - newServer := fmt.Sprintf("server: https://%s:%d", kubeConf.Cluster.ControlPlaneEndpoint.Address, kubeConf.Cluster.ControlPlaneEndpoint.Port) - newKubeConfigStr := strings.Replace(kubeConfigStr, oldServer, newServer, -1) - - if err := os.WriteFile(kubeConfigPath, []byte(newKubeConfigStr), 0644); err != nil { - return err - } - return nil -} - -// PatchKubeadmSecret is used to patch etcd's certs for kubeadm-certs secret. -func patchKubeadmSecret(runtime connector.Runtime) error { - externalEtcdCerts := []string{"external-etcd-ca.crt", "external-etcd.crt", "external-etcd.key"} - for _, cert := range externalEtcdCerts { - _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("/usr/local/bin/kubectl patch -n kube-system secret kubeadm-certs -p '{\\\"data\\\": {\\\"%s\\\": \\\"\\\"}}'", cert), - true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "patch kubeadm secret failed") - } - } - return nil -} diff --git a/cmd/kk/pkg/kubernetes/module.go b/cmd/kk/pkg/kubernetes/module.go deleted file mode 100644 index c948c1c0..00000000 --- a/cmd/kk/pkg/kubernetes/module.go +++ /dev/null @@ -1,754 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "fmt" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns" - "path/filepath" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes/templates" - dnsTemplates "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns/templates" -) - -type StatusModule struct { - common.KubeModule -} - -func (k *StatusModule) Init() { - k.Name = "KubernetesStatusModule" - k.Desc = "Get kubernetes cluster status" - - cluster := NewKubernetesStatus() - k.PipelineCache.GetOrSet(common.ClusterStatus, cluster) - - clusterStatus := &task.RemoteTask{ - Name: "GetClusterStatus", - Desc: "Get kubernetes cluster status", - Hosts: k.Runtime.GetHostsByRole(common.Master), - //Prepare: new(NoClusterInfo), - Action: new(GetClusterStatus), - Parallel: false, - } - - k.Tasks = []task.Interface{ - clusterStatus, - } -} - -type InstallKubeBinariesModule struct { - common.KubeModule -} - -func (i *InstallKubeBinariesModule) Init() { - i.Name = "InstallKubeBinariesModule" - i.Desc = "Install kubernetes cluster" - - syncBinary := &task.RemoteTask{ - Name: "SyncKubeBinary", - Desc: "Synchronize kubernetes binaries", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: new(SyncKubeBinary), - Parallel: true, - Retry: 2, - } - - chmodKubelet := &task.RemoteTask{ - Name: "ChmodKubelet", - Desc: "Change kubelet mode", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: new(ChmodKubelet), - Parallel: true, - Retry: 2, - } - - generateKubeletService := &task.RemoteTask{ - Name: "GenerateKubeletService", - Desc: "Generate kubelet service", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: &action.Template{ - Template: templates.KubeletService, - Dst: filepath.Join("/etc/systemd/system/", templates.KubeletService.Name()), - }, - Parallel: true, - Retry: 2, - } - - enableKubelet := &task.RemoteTask{ - Name: "EnableKubelet", - Desc: "Enable kubelet service", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: new(EnableKubelet), - Parallel: true, - Retry: 5, - } - - generateKubeletEnv := &task.RemoteTask{ - Name: "GenerateKubeletEnv", - Desc: "Generate kubelet env", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &NodeInCluster{Not: true}, - Action: new(GenerateKubeletEnv), - Parallel: true, - Retry: 2, - } - - i.Tasks = []task.Interface{ - syncBinary, - chmodKubelet, - generateKubeletService, - enableKubelet, - generateKubeletEnv, - } -} - -type InitKubernetesModule struct { - common.KubeModule -} - -func (i *InitKubernetesModule) Init() { - i.Name = "InitKubernetesModule" - i.Desc = "Init kubernetes cluster" - - generateKubeadmConfig := &task.RemoteTask{ - Name: "GenerateKubeadmConfig", - Desc: "Generate kubeadm config", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: &GenerateKubeadmConfig{ - IsInitConfiguration: true, - WithSecurityEnhancement: i.KubeConf.Arg.SecurityEnhancement, - }, - Parallel: true, - } - - generateAuditPolicy := &task.RemoteTask{ - Name: "GenerateAduitPolicy", - Desc: "Generate audit policy", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.EnableAudit), - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.AuditPolicy, - Dst: filepath.Join("/etc/kubernetes/audit", templates.AuditPolicy.Name()), - }, - Parallel: true, - Retry: 2, - } - - generateAuditWebhook := &task.RemoteTask{ - Name: "GenerateAduitWebhook", - Desc: "Generate audit webhook", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.EnableAudit), - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: &action.Template{ - Template: templates.AuditWebhook, - Dst: filepath.Join("/etc/kubernetes/audit", templates.AuditWebhook.Name()), - }, - Parallel: true, - Retry: 2, - } - - kubeadmInit := &task.RemoteTask{ - Name: "KubeadmInit", - Desc: "Init cluster using kubeadm", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(KubeadmInit), - Retry: 3, - Parallel: true, - } - - copyKubeConfig := &task.RemoteTask{ - Name: "CopyKubeConfig", - Desc: "Copy admin.conf to ~/.kube/config", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - }, - Action: new(CopyKubeConfigForControlPlane), - Parallel: true, - } - - removeMasterTaint := &task.RemoteTask{ - Name: "RemoveMasterTaint", - Desc: "Remove master taint", - Hosts: i.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &ClusterIsExist{Not: true}, - new(common.IsWorker), - }, - Action: new(RemoveMasterTaint), - Parallel: true, - Retry: 5, - } - - i.Tasks = []task.Interface{ - generateKubeadmConfig, - generateAuditPolicy, - generateAuditWebhook, - kubeadmInit, - copyKubeConfig, - removeMasterTaint, - } -} - -type JoinNodesModule struct { - common.KubeModule -} - -func (j *JoinNodesModule) Init() { - j.Name = "JoinNodesModule" - j.Desc = "Join kubernetes nodes" - - j.PipelineCache.Set(common.ClusterExist, true) - - generateKubeadmConfig := &task.RemoteTask{ - Name: "GenerateKubeadmConfig", - Desc: "Generate kubeadm config", - Hosts: j.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: &GenerateKubeadmConfig{ - IsInitConfiguration: false, - WithSecurityEnhancement: j.KubeConf.Arg.SecurityEnhancement, - }, - Parallel: true, - } - - generateAuditPolicy := &task.RemoteTask{ - Name: "GenerateAduitPolicy", - Desc: "Generate audit policy", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.EnableAudit), - &NodeInCluster{Not: true}, - }, - Action: &action.Template{ - Template: templates.AuditPolicy, - Dst: filepath.Join("/etc/kubernetes/audit", templates.AuditPolicy.Name()), - }, - Parallel: true, - Retry: 2, - } - - generateAuditWebhook := &task.RemoteTask{ - Name: "GenerateAduitWebhook", - Desc: "Generate audit webhook", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.EnableAudit), - &NodeInCluster{Not: true}, - }, - Action: &action.Template{ - Template: templates.AuditWebhook, - Dst: filepath.Join("/etc/kubernetes/audit", templates.AuditWebhook.Name()), - }, - Parallel: true, - Retry: 2, - } - - joinMasterNode := &task.RemoteTask{ - Name: "JoinControlPlaneNode", - Desc: "Join control-plane node", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(JoinNode), - Parallel: true, - Retry: 5, - } - - joinWorkerNode := &task.RemoteTask{ - Name: "JoinWorkerNode", - Desc: "Join worker node", - Hosts: j.Runtime.GetHostsByRole(common.Worker), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - new(common.OnlyWorker), - }, - Action: new(JoinNode), - Parallel: true, - Retry: 5, - } - - copyKubeConfig := &task.RemoteTask{ - Name: "copyKubeConfig", - Desc: "Copy admin.conf to ~/.kube/config", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - }, - Action: new(CopyKubeConfigForControlPlane), - Parallel: true, - Retry: 2, - } - - removeMasterTaint := &task.RemoteTask{ - Name: "RemoveMasterTaint", - Desc: "Remove master taint", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - &NodeInCluster{Not: true}, - new(common.IsWorker), - }, - Action: new(RemoveMasterTaint), - Parallel: true, - Retry: 5, - } - - addWorkerLabelToNode := &task.RemoteTask{ - Name: "addWorkerLabelToNode", - Desc: "Add worker label to all nodes", - Hosts: j.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(AddWorkerLabel), - Retry: 3, - } - - j.Tasks = []task.Interface{ - generateKubeadmConfig, - generateAuditPolicy, - generateAuditWebhook, - joinMasterNode, - joinWorkerNode, - copyKubeConfig, - removeMasterTaint, - addWorkerLabelToNode, - } -} - -type ResetClusterModule struct { - common.KubeModule -} - -func (r *ResetClusterModule) Init() { - r.Name = "ResetClusterModule" - r.Desc = "Reset kubernetes cluster" - - kubeadmReset := &task.RemoteTask{ - Name: "KubeadmReset", - Desc: "Reset the cluster using kubeadm", - Hosts: r.Runtime.GetHostsByRole(common.K8s), - Action: new(KubeadmReset), - Parallel: true, - } - - r.Tasks = []task.Interface{ - kubeadmReset, - } -} - -type CompareConfigAndClusterInfoModule struct { - common.KubeModule -} - -func (c *CompareConfigAndClusterInfoModule) Init() { - c.Name = "CompareConfigAndClusterInfoModule" - c.Desc = "Compare config and cluster nodes info" - - check := &task.RemoteTask{ - Name: "FindNode", - Desc: "Find information about nodes that are expected to be deleted", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - //Action: new(FindNode), - Action: new(FilterFirstMaster), - } - - c.Tasks = []task.Interface{ - check, - } -} - -type RestartKubeletModule struct { - common.KubeModule -} - -func (r *RestartKubeletModule) init() { - r.Name = "RestartKubeletModule" - r.Desc = "restart node kubelet service " - restart := &task.RemoteTask{ - Name: "RestartKubelet", - Desc: "Restart kubelet service", - Hosts: r.Runtime.GetHostsByRole(common.Master), - Action: new(RestartKubelet), - Retry: 5, - } - - r.Tasks = []task.Interface{ - restart, - } -} - -type DeleteKubeNodeModule struct { - common.KubeModule -} - -func (d *DeleteKubeNodeModule) Init() { - d.Name = "DeleteKubeNodeModule" - d.Desc = "Delete kubernetes node" - - drain := &task.RemoteTask{ - Name: "DrainNode", - Desc: "Node safely evict all pods", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(DrainNode), - Retry: 2, - } - - deleteNode := &task.RemoteTask{ - Name: "DeleteNode", - Desc: "Delete the node using kubectl", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(KubectlDeleteNode), - Retry: 5, - } - - d.Tasks = []task.Interface{ - drain, - deleteNode, - } -} - -type SetUpgradePlanModule struct { - common.KubeModule - Step UpgradeStep -} - -func (s *SetUpgradePlanModule) Init() { - s.Name = fmt.Sprintf("SetUpgradePlanModule %d/%d", s.Step, len(UpgradeStepList)) - s.Desc = "Set upgrade plan" - - plan := &task.LocalTask{ - Name: "SetUpgradePlan", - Desc: "Set upgrade plan", - Action: &SetUpgradePlan{Step: s.Step}, - } - - generateKubeadmConfigInit := &task.RemoteTask{ - Name: "GenerateKubeadmConfig", - Desc: "Generate kubeadm config", - Hosts: s.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - }, - Action: &GenerateKubeadmConfig{ - IsInitConfiguration: true, - WithSecurityEnhancement: s.KubeConf.Arg.SecurityEnhancement, - }, - Parallel: true, - } - - s.Tasks = []task.Interface{ - plan, - generateKubeadmConfigInit, - } -} - -type ProgressiveUpgradeModule struct { - common.KubeModule - Step UpgradeStep -} - -func (p *ProgressiveUpgradeModule) Init() { - p.Name = fmt.Sprintf("ProgressiveUpgradeModule %d/%d", p.Step, len(UpgradeStepList)) - p.Desc = fmt.Sprintf("Progressive upgrade %d/%d", p.Step, len(UpgradeStepList)) - - nextVersion := &task.LocalTask{ - Name: "CalculateNextVersion", - Desc: "Calculate next upgrade version", - Prepare: new(NotEqualPlanVersion), - Action: new(CalculateNextVersion), - } - - download := &task.LocalTask{ - Name: "DownloadBinaries", - Desc: "Download installation binaries", - Prepare: new(NotEqualPlanVersion), - Action: new(binaries.Download), - } - - pull := &task.RemoteTask{ - Name: "PullImages", - Desc: "Start to pull images on all nodes", - Hosts: p.Runtime.GetHostsByRole(common.K8s), - Prepare: new(NotEqualPlanVersion), - Action: new(images.PullImage), - Parallel: true, - } - - syncBinary := &task.RemoteTask{ - Name: "SyncKubeBinary", - Desc: "Synchronize kubernetes binaries", - Hosts: p.Runtime.GetHostsByRole(common.K8s), - Prepare: new(NotEqualPlanVersion), - Action: new(SyncKubeBinary), - Parallel: true, - Retry: 2, - } - - upgradeKubeMaster := &task.RemoteTask{ - Name: "UpgradeClusterOnMaster", - Desc: "Upgrade cluster on master", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: new(NotEqualPlanVersion), - Action: &UpgradeKubeMaster{ModuleName: p.Name}, - Parallel: false, - } - - cluster := NewKubernetesStatus() - p.PipelineCache.GetOrSet(common.ClusterStatus, cluster) - - clusterStatus := &task.RemoteTask{ - Name: "GetClusterStatus", - Desc: "Get kubernetes cluster status", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: new(NotEqualPlanVersion), - Action: new(GetClusterStatus), - Parallel: false, - } - - generateCoreDNS := &task.RemoteTask{ - Name: "GenerateCoreDNS", - Desc: "Generate coredns manifests", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - }, - Action: new(dns.GenerateCorednsmanifests), - Parallel: true, - } - - applyCoredns := &task.RemoteTask{ - Name: "DeployCoreDNS", - Desc: "Deploy coredns", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - }, - Action: new(dns.DeployCoreDNS), - Parallel: true, - } - - generateNodeLocalDNS := &task.RemoteTask{ - Name: "GenerateNodeLocalDNS", - Desc: "Generate nodelocaldns", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(dns.EnableNodeLocalDNS), - }, - Action: &action.Template{ - Template: dnsTemplates.NodeLocalDNSService, - Dst: filepath.Join(common.KubeConfigDir, dnsTemplates.NodeLocalDNSService.Name()), - Data: util.Data{ - "NodelocaldnsImage": images.GetImage(p.Runtime, p.KubeConf, "k8s-dns-node-cache").ImageName(), - "DNSEtcHosts": p.KubeConf.Cluster.DNS.DNSEtcHosts, - }, - }, - Parallel: true, - } - - applyNodeLocalDNS := &task.RemoteTask{ - Name: "DeployNodeLocalDNS", - Desc: "Deploy nodelocaldns", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(dns.EnableNodeLocalDNS)}, - Action: new(dns.DeployNodeLocalDNS), - Parallel: true, - Retry: 5, - } - - upgradeKubeWorker := &task.RemoteTask{ - Name: "UpgradeClusterOnWorker", - Desc: "Upgrade cluster on worker", - Hosts: p.Runtime.GetHostsByRole(common.Worker), - Prepare: &prepare.PrepareCollection{ - new(NotEqualPlanVersion), - new(common.OnlyWorker), - }, - Action: &UpgradeKubeWorker{ModuleName: p.Name}, - Parallel: false, - } - - currentVersion := &task.LocalTask{ - Name: "SetCurrentK8sVersion", - Desc: "Set current k8s version", - Prepare: new(NotEqualPlanVersion), - Action: new(SetCurrentK8sVersion), - } - - p.Tasks = []task.Interface{ - nextVersion, - download, - pull, - syncBinary, - upgradeKubeMaster, - clusterStatus, - upgradeKubeWorker, - generateCoreDNS, - applyCoredns, - generateNodeLocalDNS, - applyNodeLocalDNS, - currentVersion, - } -} - -func (p *ProgressiveUpgradeModule) Until() (*bool, error) { - f := false - t := true - currentVersion, ok := p.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return &f, errors.New("get current Kubernetes version failed by pipeline cache") - } - planVersion, ok := p.PipelineCache.GetMustString(common.PlanK8sVersion) - if !ok { - return &f, errors.New("get upgrade plan Kubernetes version failed by pipeline cache") - } - - if currentVersion != planVersion { - return &f, nil - } else { - originalDesired, ok := p.PipelineCache.GetMustString(common.DesiredK8sVersion) - if !ok { - return &f, errors.New("get original desired Kubernetes version failed by pipeline cache") - } - p.KubeConf.Cluster.Kubernetes.Version = originalDesired - return &t, nil - } -} - -type SaveKubeConfigModule struct { - common.KubeModule -} - -func (s *SaveKubeConfigModule) Init() { - s.Name = "SaveKubeConfigModule" - s.Desc = "Save kube config file as a configmap" - - save := &task.LocalTask{ - Name: "SaveKubeConfig", - Desc: "Save kube config as a configmap", - Action: new(SaveKubeConfig), - Retry: 5, - } - - s.Tasks = []task.Interface{ - save, - } -} - -type ConfigureKubernetesModule struct { - common.KubeModule -} - -func (c *ConfigureKubernetesModule) Init() { - c.Name = "ConfigureKubernetesModule" - c.Desc = "Configure kubernetes" - - configure := &task.RemoteTask{ - Name: "ConfigureKubernetes", - Desc: "Configure kubernetes", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(ConfigureKubernetes), - Retry: 3, - } - - c.Tasks = []task.Interface{ - configure, - } -} - -type SecurityEnhancementModule struct { - common.KubeModule - Skip bool -} - -func (s *SecurityEnhancementModule) IsSkip() bool { - return s.Skip -} - -func (s *SecurityEnhancementModule) Init() { - s.Name = "SecurityEnhancementModule" - s.Desc = "Security enhancement for the cluster" - - etcdSecurityEnhancement := &task.RemoteTask{ - Name: "EtcdSecurityEnhancementTask", - Desc: "Security enhancement for etcd", - Hosts: s.Runtime.GetHostsByRole(common.ETCD), - Action: new(EtcdSecurityEnhancemenAction), - Parallel: true, - } - - masterSecurityEnhancement := &task.RemoteTask{ - Name: "K8sSecurityEnhancementTask", - Desc: "Security enhancement for kubernetes", - Hosts: s.Runtime.GetHostsByRole(common.Master), - Action: new(MasterSecurityEnhancemenAction), - Parallel: true, - } - - nodesSecurityEnhancement := &task.RemoteTask{ - Name: "K8sSecurityEnhancementTask", - Desc: "Security enhancement for kubernetes", - Hosts: s.Runtime.GetHostsByRole(common.Worker), - Action: new(NodesSecurityEnhancemenAction), - Parallel: true, - } - - s.Tasks = []task.Interface{ - etcdSecurityEnhancement, - masterSecurityEnhancement, - nodesSecurityEnhancement, - } -} diff --git a/cmd/kk/pkg/kubernetes/prepares.go b/cmd/kk/pkg/kubernetes/prepares.go deleted file mode 100644 index c0f2e530..00000000 --- a/cmd/kk/pkg/kubernetes/prepares.go +++ /dev/null @@ -1,133 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type NoClusterInfo struct { - common.KubePrepare -} - -func (n *NoClusterInfo) PreCheck(_ connector.Runtime) (bool, error) { - if v, ok := n.PipelineCache.Get(common.ClusterStatus); ok { - cluster := v.(*KubernetesStatus) - if cluster.ClusterInfo == "" { - return true, nil - } - } else { - return false, errors.New("get kubernetes cluster status by pipeline cache failed") - } - return false, nil -} - -type NodeInCluster struct { - common.KubePrepare - Not bool -} - -func (n *NodeInCluster) PreCheck(runtime connector.Runtime) (bool, error) { - host := runtime.RemoteHost() - if v, ok := n.PipelineCache.Get(common.ClusterStatus); ok { - cluster := v.(*KubernetesStatus) - var versionOk bool - if res, ok := cluster.NodesInfo[host.GetName()]; ok && res != "" { - versionOk = true - } - _, ipOk := cluster.NodesInfo[host.GetInternalAddress()] - if n.Not { - return !(versionOk || ipOk), nil - } - return versionOk || ipOk, nil - } else { - return false, errors.New("get kubernetes cluster status by pipeline cache failed") - } -} - -type ClusterIsExist struct { - common.KubePrepare - Not bool -} - -func (c *ClusterIsExist) PreCheck(_ connector.Runtime) (bool, error) { - if exist, ok := c.PipelineCache.GetMustBool(common.ClusterExist); ok { - if c.Not { - return !exist, nil - } - return exist, nil - } else { - return false, errors.New("get kubernetes cluster status by pipeline cache failed") - } -} - -type NotEqualPlanVersion struct { - common.KubePrepare -} - -func (n *NotEqualPlanVersion) PreCheck(runtime connector.Runtime) (bool, error) { - planVersion, ok := n.PipelineCache.GetMustString(common.PlanK8sVersion) - if !ok { - return false, errors.New("get upgrade plan Kubernetes version failed by pipeline cache") - } - - currentVersion, ok := n.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return false, errors.New("get cluster Kubernetes version failed by pipeline cache") - } - if currentVersion == planVersion { - return false, nil - } - return true, nil -} - -type ClusterNotEqualDesiredVersion struct { - common.KubePrepare -} - -func (c *ClusterNotEqualDesiredVersion) PreCheck(runtime connector.Runtime) (bool, error) { - clusterK8sVersion, ok := c.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return false, errors.New("get cluster Kubernetes version failed by pipeline cache") - } - - if c.KubeConf.Cluster.Kubernetes.Version == clusterK8sVersion { - return false, nil - } - return true, nil -} - -type NotEqualDesiredVersion struct { - common.KubePrepare -} - -func (n *NotEqualDesiredVersion) PreCheck(runtime connector.Runtime) (bool, error) { - host := runtime.RemoteHost() - - nodeK8sVersion, ok := host.GetCache().GetMustString(common.NodeK8sVersion) - if !ok { - return false, errors.New("get node Kubernetes version failed by host cache") - } - - if n.KubeConf.Cluster.Kubernetes.Version == nodeK8sVersion { - return false, nil - } - return true, nil -} diff --git a/cmd/kk/pkg/kubernetes/task_test.go b/cmd/kk/pkg/kubernetes/task_test.go deleted file mode 100644 index 8fdf21da..00000000 --- a/cmd/kk/pkg/kubernetes/task_test.go +++ /dev/null @@ -1,73 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "testing" -) - -func Test_calculateNextStr(t *testing.T) { - tests := []struct { - currentVersion string - desiredVersion string - want string - wantErr bool - errMsg string - }{ - { - currentVersion: "v1.21.5", - desiredVersion: "v1.22.5", - want: "v1.22.5", - wantErr: false, - }, - { - currentVersion: "v1.21.5", - desiredVersion: "v1.23.5", - want: "v1.22.12", - wantErr: false, - }, - { - currentVersion: "v1.17.5", - desiredVersion: "v1.18.5", - want: "", - wantErr: true, - errMsg: "the target version v1.18.5 is not supported", - }, - { - currentVersion: "v1.17.5", - desiredVersion: "v1.21.5", - want: "", - wantErr: true, - errMsg: "Kubernetes minor version v1.18.x is not supported", - }, - } - for _, tt := range tests { - t.Run("", func(t *testing.T) { - got, err := calculateNextStr(tt.currentVersion, tt.desiredVersion) - if (err != nil) != tt.wantErr { - t.Errorf("calculateNextStr() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("calculateNextStr() got = %v, want %v", got, tt.want) - } - if err != nil && err.Error() != tt.errMsg { - t.Errorf("calculateNextStr() error = %v, want %v", err, tt.errMsg) - } - }) - } -} diff --git a/cmd/kk/pkg/kubernetes/tasks.go b/cmd/kk/pkg/kubernetes/tasks.go deleted file mode 100644 index cc14106e..00000000 --- a/cmd/kk/pkg/kubernetes/tasks.go +++ /dev/null @@ -1,1240 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "context" - "encoding/base64" - "fmt" - "os" - "path/filepath" - "sort" - "strings" - "time" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - kubeerrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - versionutil "k8s.io/apimachinery/pkg/util/version" - kube "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - - kubekeyv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/etcd" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes/templates" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -type GetClusterStatus struct { - common.KubeAction -} - -func (g *GetClusterStatus) Execute(runtime connector.Runtime) error { - exist, err := runtime.GetRunner().FileExist("/etc/kubernetes/admin.conf") - if err != nil { - return err - } - - if !exist { - g.PipelineCache.Set(common.ClusterExist, false) - return nil - } else { - g.PipelineCache.Set(common.ClusterExist, true) - - if v, ok := g.PipelineCache.Get(common.ClusterStatus); ok { - cluster := v.(*KubernetesStatus) - if err := cluster.SearchVersion(runtime); err != nil { - return err - } - if err := cluster.SearchKubeConfig(runtime); err != nil { - return err - } - if err := cluster.LoadKubeConfig(runtime, g.KubeConf); err != nil { - return err - } - if err := cluster.SearchClusterInfo(runtime); err != nil { - return err - } - if err := cluster.SearchNodesInfo(runtime); err != nil { - return err - } - if err := cluster.SearchJoinInfo(runtime); err != nil { - return err - } - - g.PipelineCache.Set(common.ClusterStatus, cluster) - } else { - return errors.New("get kubernetes cluster status by pipeline cache failed") - } - } - return nil -} - -type SyncKubeBinary struct { - common.KubeAction -} - -func (i *SyncKubeBinary) Execute(runtime connector.Runtime) error { - binariesMapObj, ok := i.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch()) - if !ok { - return errors.New("get KubeBinary by pipeline cache failed") - } - binariesMap := binariesMapObj.(map[string]*files.KubeBinary) - - if err := SyncKubeBinaries(i, runtime, binariesMap); err != nil { - return err - } - return nil -} - -// SyncKubeBinaries is used to sync kubernetes' binaries to each node. -func SyncKubeBinaries(i *SyncKubeBinary, runtime connector.Runtime, binariesMap map[string]*files.KubeBinary) error { - if err := utils.ResetTmpDir(runtime); err != nil { - return err - } - - binaryList := []string{"kubeadm", "kubelet", "kubectl", "helm", "kubecni"} - if i.KubeConf.Cluster.Network.Plugin == "calico" { - binaryList = append(binaryList, "calicoctl") - } - for _, name := range binaryList { - binary, ok := binariesMap[name] - if !ok { - return fmt.Errorf("get kube binary %s info failed: no such key", name) - } - - fileName := binary.FileName - switch name { - //case "kubelet": - // if err := runtime.GetRunner().Scp(binary.Path, fmt.Sprintf("%s/%s", common.TmpDir, binary.Name)); err != nil { - // return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed")) - // } - case "kubecni": - dst := filepath.Join(common.TmpDir, fileName) - if err := runtime.GetRunner().Scp(binary.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed")) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("tar -zxf %s -C /opt/cni/bin", dst), false); err != nil { - return err - } - default: - dst := filepath.Join(common.BinDir, fileName) - if err := runtime.GetRunner().SudoScp(binary.Path(), dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync kube binaries failed")) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("chmod +x %s", dst), false); err != nil { - return err - } - } - } - return nil -} - -type ChmodKubelet struct { - common.KubeAction -} - -func (c *ChmodKubelet) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("chmod +x /usr/local/bin/kubelet", false); err != nil { - return errors.Wrap(errors.WithStack(err), "change kubelet mode failed") - } - return nil -} - -type EnableKubelet struct { - common.KubeAction -} - -func (e *EnableKubelet) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("systemctl disable kubelet "+ - "&& systemctl enable kubelet "+ - "&& ln -snf /usr/local/bin/kubelet /usr/bin/kubelet", false); err != nil { - return errors.Wrap(errors.WithStack(err), "enable kubelet service failed") - } - return nil -} - -type GenerateKubeletEnv struct { - common.KubeAction -} - -func (g *GenerateKubeletEnv) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - templateAction := action.Template{ - Template: templates.KubeletEnv, - Dst: filepath.Join("/etc/systemd/system/kubelet.service.d", templates.KubeletEnv.Name()), - Data: util.Data{ - "NodeIP": host.GetInternalAddress(), - "Hostname": host.GetName(), - "ContainerRuntime": "", - "KubeletArgs": g.KubeConf.Cluster.Kubernetes.KubeletArgs, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type GenerateKubeadmConfig struct { - common.KubeAction - IsInitConfiguration bool - WithSecurityEnhancement bool -} - -func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - localConfig := filepath.Join(runtime.GetWorkDir(), "kubeadm-config.yaml") - if util.IsExist(localConfig) { - // todo: if it is necessary? - if err := runtime.GetRunner().SudoScp(localConfig, "/etc/kubernetes/kubeadm-config.yaml"); err != nil { - return errors.Wrap(errors.WithStack(err), "scp local kubeadm config failed") - } - } else { - // generate etcd configuration - var externalEtcd kubekeyv1alpha2.ExternalEtcd - var endpointsList, etcdCertSANs []string - - switch g.KubeConf.Cluster.Etcd.Type { - case kubekeyv1alpha2.KubeKey: - for _, host := range runtime.GetHostsByRole(common.ETCD) { - endpoint := fmt.Sprintf("https://%s:%s", host.GetInternalAddress(), kubekeyv1alpha2.DefaultEtcdPort) - endpointsList = append(endpointsList, endpoint) - } - externalEtcd.Endpoints = endpointsList - - externalEtcd.CAFile = "/etc/ssl/etcd/ssl/ca.pem" - externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s.pem", host.GetName()) - externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/node-%s-key.pem", host.GetName()) - case kubekeyv1alpha2.External: - externalEtcd.Endpoints = g.KubeConf.Cluster.Etcd.External.Endpoints - - if len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 && len(g.KubeConf.Cluster.Etcd.External.CAFile) != 0 { - externalEtcd.CAFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CAFile)) - externalEtcd.CertFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.CertFile)) - externalEtcd.KeyFile = fmt.Sprintf("/etc/ssl/etcd/ssl/%s", filepath.Base(g.KubeConf.Cluster.Etcd.External.KeyFile)) - } - case kubekeyv1alpha2.Kubeadm: - altNames := etcd.GenerateAltName(g.KubeConf, &runtime) - etcdCertSANs = append(etcdCertSANs, altNames.DNSNames...) - for _, ip := range altNames.IPs { - etcdCertSANs = append(etcdCertSANs, string(ip)) - } - } - - _, ApiServerArgs := util.GetArgs(templates.GetApiServerArgs(g.WithSecurityEnhancement, g.KubeConf.Cluster.Kubernetes.EnableAudit()), g.KubeConf.Cluster.Kubernetes.ApiServerArgs) - _, ControllerManagerArgs := util.GetArgs(templates.GetControllermanagerArgs(g.KubeConf.Cluster.Kubernetes.Version, g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.ControllerManagerArgs) - _, SchedulerArgs := util.GetArgs(templates.GetSchedulerArgs(g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.SchedulerArgs) - - checkCgroupDriver, err := templates.GetKubeletCgroupDriver(runtime, g.KubeConf) - if err != nil { - return err - } - - var ( - bootstrapToken, certificateKey string - // todo: if port needed - ) - if !g.IsInitConfiguration { - if v, ok := g.PipelineCache.Get(common.ClusterStatus); ok { - cluster := v.(*KubernetesStatus) - bootstrapToken = cluster.BootstrapToken - certificateKey = cluster.CertificateKey - } else { - return errors.New("get kubernetes cluster status by pipeline cache failed") - } - } - - templateAction := action.Template{ - Template: templates.KubeadmConfig, - Dst: filepath.Join(common.KubeConfigDir, templates.KubeadmConfig.Name()), - Data: util.Data{ - "IsInitCluster": g.IsInitConfiguration, - "ImageRepo": strings.TrimSuffix(images.GetImage(runtime, g.KubeConf, "kube-apiserver").ImageRepo(), "/kube-apiserver"), - "EtcdTypeIsKubeadm": g.KubeConf.Cluster.Etcd.Type == kubekeyv1alpha2.Kubeadm, - "EtcdCertSANs": etcdCertSANs, - "EtcdRepo": strings.TrimSuffix(images.GetImage(runtime, g.KubeConf, "etcd").ImageRepo(), "/etcd"), - "EtcdTag": images.GetImage(runtime, g.KubeConf, "etcd").Tag, - "CorednsRepo": strings.TrimSuffix(images.GetImage(runtime, g.KubeConf, "coredns").ImageRepo(), "/coredns"), - "CorednsTag": images.GetImage(runtime, g.KubeConf, "coredns").Tag, - "Version": g.KubeConf.Cluster.Kubernetes.Version, - "ClusterName": g.KubeConf.Cluster.Kubernetes.ClusterName, - "DNSDomain": g.KubeConf.Cluster.Kubernetes.DNSDomain, - "AdvertiseAddress": host.GetInternalAddress(), - "BindPort": kubekeyv1alpha2.DefaultApiserverPort, - "ControlPlaneEndpoint": fmt.Sprintf("%s:%d", g.KubeConf.Cluster.ControlPlaneEndpoint.Domain, g.KubeConf.Cluster.ControlPlaneEndpoint.Port), - "PodSubnet": g.KubeConf.Cluster.Network.KubePodsCIDR, - "ServiceSubnet": g.KubeConf.Cluster.Network.KubeServiceCIDR, - "CertSANs": g.KubeConf.Cluster.GenerateCertSANs(), - "ExternalEtcd": externalEtcd, - "NodeCidrMaskSize": g.KubeConf.Cluster.Kubernetes.NodeCidrMaskSize, - "CriSock": g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, - "ApiServerArgs": templates.UpdateFeatureGatesConfiguration(ApiServerArgs, g.KubeConf), - "EnableAudit": g.KubeConf.Cluster.Kubernetes.EnableAudit(), - "ControllerManagerArgs": templates.UpdateFeatureGatesConfiguration(ControllerManagerArgs, g.KubeConf), - "SchedulerArgs": templates.UpdateFeatureGatesConfiguration(SchedulerArgs, g.KubeConf), - "KubeletConfiguration": templates.GetKubeletConfiguration(runtime, g.KubeConf, g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, g.WithSecurityEnhancement), - "KubeProxyConfiguration": templates.GetKubeProxyConfiguration(g.KubeConf), - "IsV1beta3": versionutil.MustParseSemantic(g.KubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.22.0")), - "IsControlPlane": host.IsRole(common.Master), - "CgroupDriver": checkCgroupDriver, - "BootstrapToken": bootstrapToken, - "CertificateKey": certificateKey, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - } - return nil -} - -type KubeadmInit struct { - common.KubeAction -} - -func (k *KubeadmInit) Execute(runtime connector.Runtime) error { - initCmd := "/usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl,ImagePull" - - if k.KubeConf.Cluster.Kubernetes.DisableKubeProxy { - initCmd = initCmd + " --skip-phases=addon/kube-proxy" - } - - if _, err := runtime.GetRunner().SudoCmd(initCmd, true); err != nil { - // kubeadm reset and then retry - resetCmd := "/usr/local/bin/kubeadm reset -f" - if k.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint != "" { - resetCmd = resetCmd + " --cri-socket " + k.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint - } - _, _ = runtime.GetRunner().SudoCmd(resetCmd, true) - return errors.Wrap(errors.WithStack(err), "init kubernetes cluster failed") - } - return nil -} - -type CopyKubeConfigForControlPlane struct { - common.KubeAction -} - -func (c *CopyKubeConfigForControlPlane) Execute(runtime connector.Runtime) error { - createConfigDirCmd := "mkdir -p /root/.kube" - getKubeConfigCmd := "cp -f /etc/kubernetes/admin.conf /root/.kube/config" - cmd := strings.Join([]string{createConfigDirCmd, getKubeConfigCmd}, " && ") - if _, err := runtime.GetRunner().SudoCmd(cmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "copy kube config failed") - } - - userMkdir := "mkdir -p $HOME/.kube" - if _, err := runtime.GetRunner().Cmd(userMkdir, false); err != nil { - return errors.Wrap(errors.WithStack(err), "user mkdir $HOME/.kube failed") - } - - userCopyKubeConfig := "cp -f /etc/kubernetes/admin.conf $HOME/.kube/config" - if _, err := runtime.GetRunner().SudoCmd(userCopyKubeConfig, false); err != nil { - return errors.Wrap(errors.WithStack(err), "user copy /etc/kubernetes/admin.conf to $HOME/.kube/config failed") - } - - userId, err := runtime.GetRunner().Cmd("echo $(id -u)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user id failed") - } - - userGroupId, err := runtime.GetRunner().Cmd("echo $(id -g)", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get user group id failed") - } - - chownKubeConfig := fmt.Sprintf("chown -R %s:%s $HOME/.kube", userId, userGroupId) - if _, err := runtime.GetRunner().SudoCmd(chownKubeConfig, false); err != nil { - return errors.Wrap(errors.WithStack(err), "chown user kube config failed") - } - return nil -} - -type RemoveMasterTaint struct { - common.KubeAction -} - -func (r *RemoveMasterTaint) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "/usr/local/bin/kubectl taint nodes %s node-role.kubernetes.io/master=:NoSchedule-", - runtime.RemoteHost().GetName()), true); err != nil { - logger.Log.Warning(err.Error()) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "/usr/local/bin/kubectl taint nodes %s node-role.kubernetes.io/control-plane=:NoSchedule-", - runtime.RemoteHost().GetName()), true); err != nil { - logger.Log.Warningf(err.Error()) - } - return nil -} - -type AddWorkerLabel struct { - common.KubeAction -} - -func (a *AddWorkerLabel) Execute(runtime connector.Runtime) error { - for _, host := range runtime.GetAllHosts() { - if host.IsRole(common.Worker) { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "/usr/local/bin/kubectl label --overwrite node %s node-role.kubernetes.io/worker=", - host.GetName()), true); err != nil { - return errors.Wrap(errors.WithStack(err), "add worker label failed") - } - } - } - - return nil -} - -type JoinNode struct { - common.KubeAction -} - -func (j *JoinNode) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubeadm join --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl,ImagePull", - true); err != nil { - resetCmd := "/usr/local/bin/kubeadm reset -f" - if j.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint != "" { - resetCmd = resetCmd + " --cri-socket " + j.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint - } - _, _ = runtime.GetRunner().SudoCmd(resetCmd, true) - return errors.Wrap(errors.WithStack(err), "join node failed") - } - return nil -} - -type KubeadmReset struct { - common.KubeAction -} - -func (k *KubeadmReset) Execute(runtime connector.Runtime) error { - resetCmd := "/usr/local/bin/kubeadm reset -f" - if k.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint != "" { - resetCmd = resetCmd + " --cri-socket " + k.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint - } - _, _ = runtime.GetRunner().SudoCmd(resetCmd, true) - return nil -} - -type FilterFirstMaster struct { - common.KubeAction -} - -func filterString(filter string, nodes []string) []string { - j := 0 - for _, v := range nodes { - if v != filter { - nodes[j] = v - j++ - } - } - resArr := nodes[:j] - return resArr -} - -func (f *FilterFirstMaster) Execute(runtime connector.Runtime) error { - firstMaster := runtime.GetHostsByRole(common.Master)[0].GetName() - //kubectl get node - var nodes []string - res, err := runtime.GetRunner().Cmd( - "sudo -E /usr/local/bin/kubectl get nodes | awk '{print $1}'", - true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "kubectl get nodes failed") - } - - if !strings.Contains(res, "\r\n") { - nodes = append(nodes, res) - } else { - nodes = strings.Split(res, "\r\n") - } - //nodes filter first master - resArr := filterString(firstMaster, nodes) - //nodes filter etcd nodes - for i := 0; i < len(runtime.GetHostsByRole(common.ETCD)); i++ { - etcdName := runtime.GetHostsByRole(common.ETCD)[i].GetName() - resArr = filterString(etcdName, resArr) - } - workerName := make(map[string]struct{}) - for j := 0; j < len(runtime.GetHostsByRole(common.Worker)); j++ { - workerName[runtime.GetHostsByRole(common.Worker)[j].GetName()] = struct{}{} - } - //make sure node is not the first master and etcd node name - var node string - for i := 0; i < len(resArr); i++ { - if _, ok := workerName[resArr[i]]; ok && resArr[i] == f.KubeConf.Arg.NodeName { - node = resArr[i] - break - } - } - - if node == "" { - return errors.New("" + - "1. check the node name in the config-sample.yaml\n" + - "2. check the node name in the Kubernetes cluster\n" + - "3. check the node name is the first master and etcd node name\n") - } - - f.PipelineCache.Set("dstNode", node) - return nil -} - -type FindNode struct { - common.KubeAction -} - -func (f *FindNode) Execute(runtime connector.Runtime) error { - var resArr []string - res, err := runtime.GetRunner().Cmd( - "sudo -E /usr/local/bin/kubectl get nodes | awk '$3 !~ /master|control-plane|ROLES/ {print $1}'", - true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "kubectl get nodes failed") - } - - if !strings.Contains(res, "\r\n") { - resArr = append(resArr, res) - } else { - resArr = strings.Split(res, "\r\n") - } - - workerName := make(map[string]struct{}) - for j := 0; j < len(runtime.GetHostsByRole(common.Worker)); j++ { - workerName[runtime.GetHostsByRole(common.Worker)[j].GetName()] = struct{}{} - } - - var node string - for i := 0; i < len(resArr); i++ { - if _, ok := workerName[resArr[i]]; ok && resArr[i] == f.KubeConf.Arg.NodeName { - node = resArr[i] - break - } - } - - if node == "" { - return errors.New("" + - "1. check the node name in the config-sample.yaml\n" + - "2. check the node name in the Kubernetes cluster\n" + - "3. only support to delete a worker\n") - } - - f.PipelineCache.Set("dstNode", node) - return nil -} - -type DrainNode struct { - common.KubeAction -} - -func (d *DrainNode) Execute(runtime connector.Runtime) error { - nodeName, ok := d.PipelineCache.Get("dstNode") - if !ok { - return errors.New("get dstNode failed by pipeline cache") - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "/usr/local/bin/kubectl drain %s --delete-emptydir-data --ignore-daemonsets --timeout=2m --force", nodeName), - true); err != nil { - return errors.Wrap(err, "drain the node failed") - } - return nil -} - -type KubectlDeleteNode struct { - common.KubeAction -} - -func (k *KubectlDeleteNode) Execute(runtime connector.Runtime) error { - nodeName, ok := k.PipelineCache.Get("dstNode") - if !ok { - return errors.New("get dstNode failed by pipeline cache") - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "/usr/local/bin/kubectl delete node %s", nodeName), - true); err != nil { - return errors.Wrap(err, "delete the node failed") - } - return nil -} - -type SetUpgradePlan struct { - common.KubeAction - Step UpgradeStep -} - -func (s *SetUpgradePlan) Execute(_ connector.Runtime) error { - currentVersion, ok := s.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return errors.New("get current Kubernetes version failed by pipeline cache") - } - - desiredVersion, ok := s.PipelineCache.GetMustString(common.DesiredK8sVersion) - if !ok { - return errors.New("get desired Kubernetes version failed by pipeline cache") - } - if cmp, err := versionutil.MustParseSemantic(currentVersion).Compare(desiredVersion); err != nil { - return err - } else if cmp == 1 { - logger.Log.Messagef( - common.LocalHost, - "The current version (%s) is greater than the target version (%s)", - currentVersion, desiredVersion) - os.Exit(0) - } - - if s.Step == ToV121 { - v122 := versionutil.MustParseSemantic("v1.22.0") - atLeast := versionutil.MustParseSemantic(desiredVersion).AtLeast(v122) - cmp, err := versionutil.MustParseSemantic(currentVersion).Compare("v1.21.5") - if err != nil { - return err - } - if atLeast && cmp <= 0 { - desiredVersion = "v1.21.5" - } - } - - s.PipelineCache.Set(common.PlanK8sVersion, desiredVersion) - return nil -} - -type CalculateNextVersion struct { - common.KubeAction -} - -func (c *CalculateNextVersion) Execute(_ connector.Runtime) error { - currentVersion, ok := c.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return errors.New("get current Kubernetes version failed by pipeline cache") - } - planVersion, ok := c.PipelineCache.GetMustString(common.PlanK8sVersion) - if !ok { - return errors.New("get upgrade plan Kubernetes version failed by pipeline cache") - } - nextVersionStr, err := calculateNextStr(currentVersion, planVersion) - if err != nil { - return errors.Wrap(err, "calculate next version failed") - } - c.KubeConf.Cluster.Kubernetes.Version = nextVersionStr - return nil -} - -func calculateNextStr(currentVersion, desiredVersion string) (string, error) { - current := versionutil.MustParseSemantic(currentVersion) - target := versionutil.MustParseSemantic(desiredVersion) - var nextVersionMinor uint - if target.Minor() == current.Minor() { - nextVersionMinor = current.Minor() - } else { - nextVersionMinor = current.Minor() + 1 - } - - if nextVersionMinor == target.Minor() { - if _, ok := files.FileSha256["kubeadm"]["amd64"][desiredVersion]; !ok { - return "", errors.Errorf("the target version %s is not supported", desiredVersion) - } - return desiredVersion, nil - } else { - nextVersionPatchList := make([]int, 0) - for supportVersionStr := range files.FileSha256["kubeadm"]["amd64"] { - supportVersion := versionutil.MustParseSemantic(supportVersionStr) - if supportVersion.Minor() == nextVersionMinor { - nextVersionPatchList = append(nextVersionPatchList, int(supportVersion.Patch())) - } - } - sort.Ints(nextVersionPatchList) - - nextVersion := current.WithMinor(nextVersionMinor) - if len(nextVersionPatchList) == 0 { - return "", errors.Errorf("Kubernetes minor version v%d.%d.x is not supported", nextVersion.Major(), nextVersion.Minor()) - } - nextVersion = nextVersion.WithPatch(uint(nextVersionPatchList[len(nextVersionPatchList)-1])) - - return fmt.Sprintf("v%s", nextVersion.String()), nil - } -} - -type RestartKubelet struct { - common.KubeAction - ModuleName string -} - -func (r *RestartKubelet) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - if _, err := runtime.GetRunner().SudoCmd("systemctl stop kubelet", false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("stop kubelet failed: %s", host.GetName())) - } - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && systemctl restart kubelet", true); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("restart kubelet failed: %s", host.GetName())) - } - time.Sleep(10 * time.Second) - return nil -} - -type UpgradeKubeMaster struct { - common.KubeAction - ModuleName string -} - -func (u *UpgradeKubeMaster) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - if err := KubeadmUpgradeTasks(runtime, u); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("upgrade cluster using kubeadm failed: %s", host.GetName())) - } - - if _, err := runtime.GetRunner().SudoCmd("systemctl stop kubelet", false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("stop kubelet failed: %s", host.GetName())) - } - - if versionutil.MustParseSemantic(u.KubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.24.0")) { - if _, err := runtime.GetRunner().SudoCmd("sed -i 's/ --network-plugin=cni / /g' /var/lib/kubelet/kubeadm-flags.env", false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("update kubelet config failed: %s", host.GetName())) - } - } - - if err := SetKubeletTasks(runtime, u.KubeAction); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("set kubelet failed: %s", host.GetName())) - } - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && systemctl restart kubelet", true); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("restart kubelet failed: %s", host.GetName())) - } - - time.Sleep(10 * time.Second) - return nil -} - -type UpgradeKubeWorker struct { - common.KubeAction - ModuleName string -} - -func (u *UpgradeKubeWorker) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubeadm upgrade node", true); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("upgrade node using kubeadm failed: %s", host.GetName())) - } - if _, err := runtime.GetRunner().SudoCmd("systemctl stop kubelet", true); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("stop kubelet failed: %s", host.GetName())) - } - if versionutil.MustParseSemantic(u.KubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.24.0")) { - if _, err := runtime.GetRunner().SudoCmd("sed -i 's/ --network-plugin=cni / /g' /var/lib/kubelet/kubeadm-flags.env", false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("update kubelet config failed: %s", host.GetName())) - } - } - if err := SetKubeletTasks(runtime, u.KubeAction); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("set kubelet failed: %s", host.GetName())) - } - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && systemctl restart kubelet", true); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("restart kubelet failed: %s", host.GetName())) - } - time.Sleep(10 * time.Second) - return nil -} - -func KubeadmUpgradeTasks(runtime connector.Runtime, u *UpgradeKubeMaster) error { - host := runtime.RemoteHost() - - kubeadmUpgrade := &task.RemoteTask{ - Name: "KubeadmUpgrade", - Desc: "Upgrade cluster using kubeadm", - Hosts: []connector.Host{host}, - Prepare: new(NotEqualDesiredVersion), - Action: new(KubeadmUpgrade), - Parallel: false, - Retry: 3, - } - - copyKubeConfig := &task.RemoteTask{ - Name: "CopyKubeConfig", - Desc: "Copy admin.conf to ~/.kube/config", - Hosts: []connector.Host{host}, - Prepare: new(NotEqualDesiredVersion), - Action: new(CopyKubeConfigForControlPlane), - Parallel: false, - Retry: 2, - } - - tasks := []task.Interface{ - kubeadmUpgrade, - copyKubeConfig, - } - - for i := range tasks { - t := tasks[i] - t.Init(runtime, u.ModuleCache, u.PipelineCache) - if res := t.Execute(); res.IsFailed() { - return res.CombineErr() - } - } - return nil -} - -type KubeadmUpgrade struct { - common.KubeAction -} - -func (k *KubeadmUpgrade) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - fmt.Println(k.KubeConf.Cluster.Kubernetes.Version) - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "timeout -k 600s 600s /usr/local/bin/kubeadm upgrade apply %s -y "+ - "--ignore-preflight-errors=all "+ - "--allow-experimental-upgrades "+ - "--allow-release-candidate-upgrades "+ - "--etcd-upgrade=false "+ - "--certificate-renewal=true ", - k.KubeConf.Cluster.Kubernetes.Version), false); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("upgrade master failed: %s", host.GetName())) - } - return nil -} - -func SetKubeletTasks(runtime connector.Runtime, kubeAction common.KubeAction) error { - host := runtime.RemoteHost() - chmodKubelet := &task.RemoteTask{ - Name: "ChownKubelet", - Desc: "Change kubelet owner", - Hosts: []connector.Host{host}, - Prepare: new(NotEqualDesiredVersion), - Action: new(ChmodKubelet), - Parallel: false, - Retry: 2, - } - - enableKubelet := &task.RemoteTask{ - Name: "EnableKubelet", - Desc: "enable kubelet service", - Hosts: []connector.Host{host}, - Prepare: new(NotEqualDesiredVersion), - Action: new(EnableKubelet), - Parallel: false, - Retry: 5, - } - - tasks := []task.Interface{ - chmodKubelet, - enableKubelet, - } - - for i := range tasks { - t := tasks[i] - t.Init(runtime, kubeAction.ModuleCache, kubeAction.PipelineCache) - if res := t.Execute(); res.IsFailed() { - return res.CombineErr() - } - } - return nil -} - -//type UpgradeDNS struct { -// common.KubeAction -// ModuleName string -//} -// -//func (r *UpgradeDNS) Execute(runtime connector.Runtime) error { -// if err := UpgradeCoredns(runtime, r.KubeAction); err != nil { -// return errors.Wrap(errors.WithStack(err), "re-config coredns failed") -// } -// return nil -//} -// -//func UpgradeCoredns(runtime connector.Runtime, kubeAction common.KubeAction) error { -// host := runtime.RemoteHost() -// -// generateCoreDNSSvc := &task.RemoteTask{ -// Name: "GenerateCoreDNS", -// Desc: "generate coredns manifests", -// Hosts: []connector.Host{host}, -// Prepare: &prepare.PrepareCollection{ -// new(common.OnlyFirstMaster), -// }, -// Action: &action.Template{ -// Template: dnsTemplates.Coredns, -// Dst: filepath.Join(common.KubeConfigDir, dnsTemplates.Coredns.Name()), -// Data: util.Data{ -// "ClusterIP": kubeAction.KubeConf.Cluster.CorednsClusterIP(), -// "CorednsImage": images.GetImage(runtime, kubeAction.KubeConf, "coredns").ImageName(), -// "DNSEtchHsts": kubeAction.KubeConf.Cluster.DNS.DNSEtcHosts, -// }, -// }, -// Parallel: true, -// } -// -// override := &task.RemoteTask{ -// Name: "UpgradeCoreDNS", -// Desc: "upgrade coredns", -// Hosts: []connector.Host{host}, -// Prepare: &prepare.PrepareCollection{ -// new(common.OnlyFirstMaster), -// }, -// Action: new(dns.DeployCoreDNS), -// Parallel: false, -// } -// -// generateNodeLocalDNS := &task.RemoteTask{ -// Name: "GenerateNodeLocalDNS", -// Desc: "generate nodelocaldns", -// Hosts: []connector.Host{host}, -// Prepare: &prepare.PrepareCollection{ -// new(common.OnlyFirstMaster), -// new(dns.EnableNodeLocalDNS), -// }, -// Action: &action.Template{ -// Template: dnsTemplates.NodeLocalDNSService, -// Dst: filepath.Join(common.KubeConfigDir, dnsTemplates.NodeLocalDNSService.Name()), -// Data: util.Data{ -// "NodelocaldnsImage": images.GetImage(runtime, kubeAction.KubeConf, "k8s-dns-node-cache").ImageName(), -// }, -// }, -// Parallel: true, -// } -// -// applyNodeLocalDNS := &task.RemoteTask{ -// Name: "DeployNodeLocalDNS", -// Desc: "deploy nodelocaldns", -// Hosts: []connector.Host{host}, -// Prepare: &prepare.PrepareCollection{ -// new(common.OnlyFirstMaster), -// new(dns.EnableNodeLocalDNS), -// }, -// Action: new(dns.DeployNodeLocalDNS), -// Parallel: true, -// Retry: 5, -// } -// -// generateNodeLocalDNSConfigMap := &task.RemoteTask{ -// Name: "GenerateNodeLocalDNSConfigMap", -// Desc: "generate nodelocaldns configmap", -// Hosts: []connector.Host{host}, -// Prepare: &prepare.PrepareCollection{ -// new(common.OnlyFirstMaster), -// new(dns.EnableNodeLocalDNS), -// new(dns.NodeLocalDNSConfigMapNotExist), -// }, -// Action: new(dns.GenerateNodeLocalDNSConfigMap), -// Parallel: true, -// } -// -// applyNodeLocalDNSConfigMap := &task.RemoteTask{ -// Name: "ApplyNodeLocalDNSConfigMap", -// Desc: "apply nodelocaldns configmap", -// Hosts: []connector.Host{host}, -// Prepare: &prepare.PrepareCollection{ -// new(common.OnlyFirstMaster), -// new(dns.EnableNodeLocalDNS), -// new(dns.NodeLocalDNSConfigMapNotExist), -// }, -// Action: new(dns.ApplyNodeLocalDNSConfigMap), -// Parallel: true, -// Retry: 5, -// } -// -// tasks := []task.Interface{ -// override, -// generateCoreDNSSvc, -// override, -// generateNodeLocalDNS, -// applyNodeLocalDNS, -// generateNodeLocalDNSConfigMap, -// applyNodeLocalDNSConfigMap, -// } -// -// for i := range tasks { -// t := tasks[i] -// t.Init(runtime, kubeAction.ModuleCache, kubeAction.PipelineCache) -// if res := t.Execute(); res.IsFailed() { -// return res.CombineErr() -// } -// } -// return nil -//} - -type SetCurrentK8sVersion struct { - common.KubeAction -} - -func (s *SetCurrentK8sVersion) Execute(_ connector.Runtime) error { - s.PipelineCache.Set(common.K8sVersion, s.KubeConf.Cluster.Kubernetes.Version) - return nil -} - -type SaveKubeConfig struct { - common.KubeAction -} - -func (s *SaveKubeConfig) Execute(runtime connector.Runtime) error { - status, ok := s.PipelineCache.Get(common.ClusterStatus) - if !ok { - return errors.New("get kubernetes status failed by pipeline cache") - } - cluster := status.(*KubernetesStatus) - kubeConfigStr := cluster.KubeConfig - - clusterPublicAddress := s.KubeConf.Cluster.ControlPlaneEndpoint.Address - master1 := runtime.GetHostsByRole(common.Master)[0] - if clusterPublicAddress == master1.GetInternalAddress() || clusterPublicAddress == "" { - clusterPublicAddress = master1.GetAddress() - } - - oldServer := fmt.Sprintf("https://%s:%d", s.KubeConf.Cluster.ControlPlaneEndpoint.Domain, s.KubeConf.Cluster.ControlPlaneEndpoint.Port) - newServer := fmt.Sprintf("https://%s:%d", clusterPublicAddress, s.KubeConf.Cluster.ControlPlaneEndpoint.Port) - newKubeConfigStr := strings.Replace(kubeConfigStr, oldServer, newServer, -1) - kubeConfigBase64 := base64.StdEncoding.EncodeToString([]byte(newKubeConfigStr)) - - config, err := clientcmd.NewClientConfigFromBytes([]byte(newKubeConfigStr)) - if err != nil { - return err - } - restConfig, err := config.ClientConfig() - if err != nil { - return err - } - clientsetForCluster, err := kube.NewForConfig(restConfig) - if err != nil { - return err - } - - namespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubekey-system", - }, - } - if _, err := clientsetForCluster. - CoreV1(). - Namespaces(). - Get(context.TODO(), namespace.Name, metav1.GetOptions{}); kubeerrors.IsNotFound(err) { - if _, err := clientsetForCluster. - CoreV1(). - Namespaces(). - Create(context.TODO(), namespace, metav1.CreateOptions{}); err != nil { - return err - } - } else { - return err - } - - cm := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-kubeconfig", s.KubeConf.ClusterName), - }, - Data: map[string]string{ - "kubeconfig": kubeConfigBase64, - }, - } - - if _, err := clientsetForCluster. - CoreV1(). - ConfigMaps("kubekey-system"). - Get(context.TODO(), cm.Name, metav1.GetOptions{}); kubeerrors.IsNotFound(err) { - if _, err := clientsetForCluster. - CoreV1(). - ConfigMaps("kubekey-system"). - Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil { - return err - } - } else { - if _, err := clientsetForCluster. - CoreV1(). - ConfigMaps("kubekey-system"). - Update(context.TODO(), cm, metav1.UpdateOptions{}); err != nil { - return err - } - } - return nil -} - -type ConfigureKubernetes struct { - common.KubeAction -} - -func (c *ConfigureKubernetes) Execute(runtime connector.Runtime) error { - hosts := runtime.GetHostsByRole(common.K8s) - - for j := 0; j < len(hosts); j++ { - kubeHost := hosts[j].(*kubekeyv1alpha2.KubeHost) - for k, v := range kubeHost.Labels { - labelCmd := fmt.Sprintf("/usr/local/bin/kubectl label --overwrite node %s %s=%s", hosts[j].GetName(), k, v) - _, err := runtime.GetRunner().SudoCmd(labelCmd, true) - if err != nil { - return err - } - } - } - return nil -} - -type EtcdSecurityEnhancemenAction struct { - common.KubeAction - ModuleName string -} - -func (s *EtcdSecurityEnhancemenAction) Execute(runtime connector.Runtime) error { - chmodEtcdCertsDirCmd := "chmod 700 /etc/ssl/etcd/ssl" - chmodEtcdCertsCmd := "chmod 600 /etc/ssl/etcd/ssl/*" - chmodEtcdDataDirCmd := "chmod 700 /var/lib/etcd" - chmodEtcdCmd := "chmod 550 /usr/local/bin/etcd*" - - chownEtcdCertsDirCmd := "chown root:root /etc/ssl/etcd/ssl" - chownEtcdCertsCmd := "chown root:root /etc/ssl/etcd/ssl/*" - chownEtcdDataDirCmd := "chown etcd:etcd /var/lib/etcd" - chownEtcdCmd := "chown root:root /usr/local/bin/etcd*" - - ETCDcmds := []string{chmodEtcdCertsDirCmd, chmodEtcdCertsCmd, chmodEtcdDataDirCmd, chmodEtcdCmd, chownEtcdCertsDirCmd, chownEtcdCertsCmd, chownEtcdDataDirCmd, chownEtcdCmd} - - if _, err := runtime.GetRunner().SudoCmd(strings.Join(ETCDcmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - - return nil -} - -type MasterSecurityEnhancemenAction struct { - common.KubeAction - ModuleName string -} - -func (k *MasterSecurityEnhancemenAction) Execute(runtime connector.Runtime) error { - // Control-plane Security Enhancemen - chmodKubernetesDirCmd := "chmod 644 /etc/kubernetes" - chownKubernetesDirCmd := "chown root:root /etc/kubernetes" - - chmodKubernetesConfigCmd := "chmod 600 -R /etc/kubernetes" - chownKubernetesConfigCmd := "chown root:root -R /etc/kubernetes/*" - - chmodKubenretesManifestsDirCmd := "chmod 644 /etc/kubernetes/manifests" - chownKubenretesManifestsDirCmd := "chown root:root /etc/kubernetes/manifests" - - chmodKubenretesCertsDirCmd := "chmod 644 /etc/kubernetes/pki" - chownKubenretesCertsDirCmd := "chown root:root /etc/kubernetes/pki" - - // node Security Enhancemen - chmodCniConfigDir := "chmod 600 -R /etc/cni/net.d" - chownCniConfigDir := "chown root:root -R /etc/cni/net.d" - - chmodBinDir := "chmod 550 /usr/local/bin/" - chownBinDir := "chown root:root /usr/local/bin/" - - chmodKubeCmd := "chmod 550 -R /usr/local/bin/kube*" - chownKubeCmd := "chown root:root -R /usr/local/bin/kube*" - - chmodHelmCmd := "chmod 550 /usr/local/bin/helm" - chownHelmCmd := "chown root:root /usr/local/bin/helm" - - chmodCniDir := "chmod 550 -R /opt/cni/bin" - chownCniDir := "chown root:root -R /opt/cni/bin" - - chmodKubeletConfig := "chmod 640 /var/lib/kubelet/config.yaml && chmod 640 -R /etc/systemd/system/kubelet.service*" - chownKubeletConfig := "chown root:root /var/lib/kubelet/config.yaml && chown root:root -R /etc/systemd/system/kubelet.service*" - - chmodCertsRenew := "chmod 640 /etc/systemd/system/k8s-certs-renew*" - chownCertsRenew := "chown root:root /etc/systemd/system/k8s-certs-renew*" - - chmodMasterCmds := []string{chmodKubernetesConfigCmd, chmodKubernetesDirCmd, chmodKubenretesManifestsDirCmd, chmodKubenretesCertsDirCmd} - if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodMasterCmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - chownMasterCmds := []string{chownKubernetesConfigCmd, chownKubernetesDirCmd, chownKubenretesManifestsDirCmd, chownKubenretesCertsDirCmd} - if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownMasterCmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - - chmodNodesCmds := []string{chmodBinDir, chmodKubeCmd, chmodHelmCmd, chmodCniDir, chmodCniConfigDir, chmodKubeletConfig, chmodCertsRenew} - if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodNodesCmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - chownNodesCmds := []string{chownBinDir, chownKubeCmd, chownHelmCmd, chownCniDir, chownCniConfigDir, chownKubeletConfig, chownCertsRenew} - if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownNodesCmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - - return nil -} - -type NodesSecurityEnhancemenAction struct { - common.KubeAction - ModuleName string -} - -func (n *NodesSecurityEnhancemenAction) Execute(runtime connector.Runtime) error { - // Control-plane Security Enhancemen - chmodKubernetesDirCmd := "chmod 644 /etc/kubernetes" - chownKubernetesDirCmd := "chown root:root /etc/kubernetes" - - chmodKubernetesConfigCmd := "chmod 600 -R /etc/kubernetes" - chownKubernetesConfigCmd := "chown root:root -R /etc/kubernetes" - - chmodKubenretesManifestsDirCmd := "chmod 644 /etc/kubernetes/manifests" - chownKubenretesManifestsDirCmd := "chown root:root /etc/kubernetes/manifests" - - chmodKubenretesCertsDirCmd := "chmod 644 /etc/kubernetes/pki" - chownKubenretesCertsDirCmd := "chown root:root /etc/kubernetes/pki" - - // node Security Enhancemen - chmodCniConfigDir := "chmod 600 -R /etc/cni/net.d" - chownCniConfigDir := "chown root:root -R /etc/cni/net.d" - - chmodBinDir := "chmod 550 /usr/local/bin/" - chownBinDir := "chown root:root /usr/local/bin/" - - chmodKubeCmd := "chmod 550 -R /usr/local/bin/kube*" - chownKubeCmd := "chown root:root -R /usr/local/bin/kube*" - - chmodHelmCmd := "chmod 550 /usr/local/bin/helm" - chownHelmCmd := "chown root:root /usr/local/bin/helm" - - chmodCniDir := "chmod 550 -R /opt/cni/bin" - chownCniDir := "chown root:root -R /opt/cni/bin" - - chmodKubeletConfig := "chmod 640 /var/lib/kubelet/config.yaml && chmod 640 -R /etc/systemd/system/kubelet.service*" - chownKubeletConfig := "chown root:root /var/lib/kubelet/config.yaml && chown root:root -R /etc/systemd/system/kubelet.service*" - - chmodMasterCmds := []string{chmodKubernetesConfigCmd, chmodKubernetesDirCmd, chmodKubenretesManifestsDirCmd, chmodKubenretesCertsDirCmd} - if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodMasterCmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - chownMasterCmds := []string{chownKubernetesConfigCmd, chownKubernetesDirCmd, chownKubenretesManifestsDirCmd, chownKubenretesCertsDirCmd} - if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownMasterCmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - - chmodNodesCmds := []string{chmodBinDir, chmodKubeCmd, chmodHelmCmd, chmodCniDir, chmodCniConfigDir, chmodKubeletConfig} - if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodNodesCmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - chownNodesCmds := []string{chownBinDir, chownKubeCmd, chownHelmCmd, chownCniDir, chownCniConfigDir, chownKubeletConfig} - if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownNodesCmds, " && "), true); err != nil { - return errors.Wrap(errors.WithStack(err), "Updating permissions failed.") - } - - return nil -} diff --git a/cmd/kk/pkg/kubernetes/templates/audit.go b/cmd/kk/pkg/kubernetes/templates/audit.go deleted file mode 100644 index 83b19e02..00000000 --- a/cmd/kk/pkg/kubernetes/templates/audit.go +++ /dev/null @@ -1,168 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -// AuditPolicy defines the template of kube-apiserver audit-policy. -var AuditPolicy = template.Must(template.New("audit-policy.yaml").Parse( - dedent.Dedent(`apiVersion: audit.k8s.io/v1 -kind: Policy -rules: - # The following requests were manually identified as high-volume and low-risk, - # so drop them. - - level: None - users: ["system:kube-proxy"] - verbs: ["watch"] - resources: - - group: "" # core - resources: ["endpoints", "services", "services/status"] - - level: None - users: ["system:unsecured"] - namespaces: ["kube-system"] - verbs: ["get"] - resources: - - group: "" # core - resources: ["configmaps"] - - level: None - users: ["kubelet"] # legacy kubelet identity - verbs: ["get"] - resources: - - group: "" # core - resources: ["nodes", "nodes/status"] - - level: None - userGroups: ["system:nodes"] - verbs: ["get"] - resources: - - group: "" # core - resources: ["nodes", "nodes/status"] - - level: None - users: - - system:kube-controller-manager - - system:kube-scheduler - - system:serviceaccount:kube-system:endpoint-controller - verbs: ["get", "update"] - namespaces: ["kube-system"] - resources: - - group: "" # core - resources: ["endpoints"] - - level: None - users: ["system:apiserver"] - verbs: ["get"] - resources: - - group: "" # core - resources: ["namespaces", "namespaces/status", "namespaces/finalize"] - # Don't log HPA fetching metrics. - - level: None - users: - - system:kube-controller-manager - verbs: ["get", "list"] - resources: - - group: "metrics.k8s.io" - # Don't log these read-only URLs. - - level: None - nonResourceURLs: - - /healthz* - - /version - - /swagger* - # Don't log events requests. - - level: None - resources: - - group: "" # core - resources: ["events"] - # Secrets, ConfigMaps, TokenRequest and TokenReviews can contain sensitive & binary data, - # so only log at the Metadata level. - - level: Metadata - resources: - - group: "" # core - resources: ["secrets", "configmaps", "serviceaccounts/token"] - - group: authentication.k8s.io - resources: ["tokenreviews"] - omitStages: - - "RequestReceived" - # Get responses can be large; skip them. - - level: Request - verbs: ["get", "list", "watch"] - resources: - - group: "" # core - - group: "admissionregistration.k8s.io" - - group: "apiextensions.k8s.io" - - group: "apiregistration.k8s.io" - - group: "apps" - - group: "authentication.k8s.io" - - group: "authorization.k8s.io" - - group: "autoscaling" - - group: "batch" - - group: "certificates.k8s.io" - - group: "extensions" - - group: "metrics.k8s.io" - - group: "networking.k8s.io" - - group: "policy" - - group: "rbac.authorization.k8s.io" - - group: "settings.k8s.io" - - group: "storage.k8s.io" - omitStages: - - "RequestReceived" - # Default level for known APIs - - level: RequestResponse - resources: - - group: "" # core - - group: "admissionregistration.k8s.io" - - group: "apiextensions.k8s.io" - - group: "apiregistration.k8s.io" - - group: "apps" - - group: "authentication.k8s.io" - - group: "authorization.k8s.io" - - group: "autoscaling" - - group: "batch" - - group: "certificates.k8s.io" - - group: "extensions" - - group: "metrics.k8s.io" - - group: "networking.k8s.io" - - group: "policy" - - group: "rbac.authorization.k8s.io" - - group: "settings.k8s.io" - - group: "storage.k8s.io" - omitStages: - - "RequestReceived" - # Default level for all other requests. - - level: Metadata - omitStages: - - "RequestReceived" - `))) - -// AuditWebhook defines the template of kube-apiserver audit-webhook. -var AuditWebhook = template.Must(template.New("audit-webhook.yaml").Parse( - dedent.Dedent(`apiVersion: v1 -kind: Config -clusters: -- name: kube-auditing - cluster: - server: https://SHOULD_BE_REPLACED:6443/audit/webhook/event - insecure-skip-tls-verify: true -contexts: -- context: - cluster: kube-auditing - user: "" - name: default-context -current-context: default-context -preferences: {} -users: [] - `))) diff --git a/cmd/kk/pkg/kubernetes/templates/kubeadm_config.go b/cmd/kk/pkg/kubernetes/templates/kubeadm_config.go deleted file mode 100644 index 17d75a0a..00000000 --- a/cmd/kk/pkg/kubernetes/templates/kubeadm_config.go +++ /dev/null @@ -1,471 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "fmt" - "strings" - "text/template" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - "gopkg.in/yaml.v3" - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" -) - -var ( - // KubeadmConfig defines the template of kubeadm configuration file. - KubeadmConfig = template.Must(template.New("kubeadm-config.yaml").Funcs(utils.FuncMap).Parse( - dedent.Dedent(` -{{- if .IsInitCluster -}} ---- -apiVersion: kubeadm.k8s.io/{{ if .IsV1beta3 }}v1beta3{{ else }}v1beta2{{ end }} -kind: ClusterConfiguration -etcd: -{{- if .EtcdTypeIsKubeadm }} - local: - imageRepository: {{ .EtcdRepo }} - imageTag: {{ .EtcdTag }} - serverCertSANs: - {{- range .ExternalEtcd.Endpoints }} - - {{ . }} - {{- end }} -{{- else }} - external: - endpoints: - {{- range .ExternalEtcd.Endpoints }} - - {{ . }} - {{- end }} -{{- if .ExternalEtcd.CAFile }} - caFile: {{ .ExternalEtcd.CAFile }} -{{- end }} -{{- if .ExternalEtcd.CertFile }} - certFile: {{ .ExternalEtcd.CertFile }} -{{- end }} -{{- if .ExternalEtcd.KeyFile }} - keyFile: {{ .ExternalEtcd.KeyFile }} -{{- end }} -{{- end }} -dns: - type: CoreDNS - imageRepository: {{ .CorednsRepo }} - imageTag: {{ .CorednsTag }} -imageRepository: {{ .ImageRepo }} -kubernetesVersion: {{ .Version }} -certificatesDir: /etc/kubernetes/pki -clusterName: {{ .ClusterName }} -controlPlaneEndpoint: {{ .ControlPlaneEndpoint }} -networking: - dnsDomain: {{ .DNSDomain }} - podSubnet: {{ .PodSubnet }} - serviceSubnet: {{ .ServiceSubnet }} -apiServer: - extraArgs: -{{ toYaml .ApiServerArgs | indent 4}} - certSANs: - {{- range .CertSANs }} - - "{{ . }}" - {{- end }} -{{- if .EnableAudit }} - extraVolumes: - - name: k8s-audit - hostPath: /etc/kubernetes/audit - mountPath: /etc/kubernetes/audit - pathType: DirectoryOrCreate -{{- end }} -controllerManager: - extraArgs: - node-cidr-mask-size: "{{ .NodeCidrMaskSize }}" -{{ toYaml .ControllerManagerArgs | indent 4 }} - extraVolumes: - - name: host-time - hostPath: /etc/localtime - mountPath: /etc/localtime - readOnly: true -scheduler: - extraArgs: -{{ toYaml .SchedulerArgs | indent 4 }} - ---- -apiVersion: kubeadm.k8s.io/{{ if .IsV1beta3 }}v1beta3{{ else }}v1beta2{{ end }} -kind: InitConfiguration -localAPIEndpoint: - advertiseAddress: {{ .AdvertiseAddress }} - bindPort: {{ .BindPort }} -nodeRegistration: -{{- if .CriSock }} - criSocket: {{ .CriSock }} -{{- end }} - kubeletExtraArgs: - cgroup-driver: {{ .CgroupDriver }} ---- -apiVersion: kubeproxy.config.k8s.io/v1alpha1 -kind: KubeProxyConfiguration -{{ toYaml .KubeProxyConfiguration }} ---- -apiVersion: kubelet.config.k8s.io/v1beta1 -kind: KubeletConfiguration -{{ toYaml .KubeletConfiguration }} - -{{- else -}} ---- -apiVersion: kubeadm.k8s.io/{{ if .IsV1beta3 }}v1beta3{{ else }}v1beta2{{ end }} -kind: JoinConfiguration -discovery: - bootstrapToken: - apiServerEndpoint: {{ .ControlPlaneEndpoint }} - token: "{{ .BootstrapToken }}" - unsafeSkipCAVerification: true - tlsBootstrapToken: "{{ .BootstrapToken }}" -{{- if .IsControlPlane }} -controlPlane: - localAPIEndpoint: - advertiseAddress: {{ .AdvertiseAddress }} - bindPort: {{ .BindPort }} - certificateKey: {{ .CertificateKey }} -{{- end }} -nodeRegistration: -{{- if .CriSock }} - criSocket: {{ .CriSock }} -{{- end }} - kubeletExtraArgs: - cgroup-driver: {{ .CgroupDriver }} - -{{- end }} - `))) -) - -var ( - // ref: https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ - FeatureGatesDefaultConfiguration = map[string]bool{ - "RotateKubeletServerCertificate": true, //k8s 1.7+ - "TTLAfterFinished": true, //k8s 1.12+ - } - FeatureGatesSecurityDefaultConfiguration = map[string]bool{ - "RotateKubeletServerCertificate": true, //k8s 1.7+ - "TTLAfterFinished": true, //k8s 1.12+ - "SeccompDefault": true, //kubelet - } - - ApiServerArgs = map[string]string{ - "bind-address": "0.0.0.0", - } - ApiServerSecurityArgs = map[string]string{ - "bind-address": "0.0.0.0", - "authorization-mode": "Node,RBAC", - // --enable-admission-plugins=EventRateLimit must have a configuration file - "enable-admission-plugins": "AlwaysPullImages,ServiceAccount,NamespaceLifecycle,NodeRestriction,LimitRanger,ResourceQuota,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,PodNodeSelector,PodSecurity", - // "audit-log-path": "/var/log/apiserver/audit.log", // need audit policy - "profiling": "false", - "request-timeout": "120s", - "service-account-lookup": "true", - "tls-min-version": "VersionTLS12", - "tls-cipher-suites": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", - } - auditArgs = map[string]string{ - "audit-log-format": "json", - "audit-log-maxbackup": "2", - "audit-log-maxsize": "200", - "audit-policy-file": "/etc/kubernetes/audit/audit-policy.yaml", - "audit-webhook-config-file": "/etc/kubernetes/audit/audit-webhook.yaml", - } - ControllermanagerArgs = map[string]string{ - "bind-address": "0.0.0.0", - "cluster-signing-duration": "87600h", - } - ControllermanagerSecurityArgs = map[string]string{ - "bind-address": "127.0.0.1", - "cluster-signing-duration": "87600h", - "profiling": "false", - "terminated-pod-gc-threshold": "50", - "use-service-account-credentials": "true", - } - SchedulerArgs = map[string]string{ - "bind-address": "0.0.0.0", - } - SchedulerSecurityArgs = map[string]string{ - "bind-address": "127.0.0.1", - "profiling": "false", - } -) - -func GetApiServerArgs(securityEnhancement bool, enableAudit bool) map[string]string { - if securityEnhancement { - if enableAudit { - for k, v := range auditArgs { - ApiServerSecurityArgs[k] = v - } - } - return ApiServerSecurityArgs - } - - if enableAudit { - for k, v := range auditArgs { - ApiServerArgs[k] = v - } - } - - return ApiServerArgs -} - -func GetControllermanagerArgs(version string, securityEnhancement bool) map[string]string { - var args map[string]string - if securityEnhancement { - args = copyStringMap(ControllermanagerSecurityArgs) - } else { - args = copyStringMap(ControllermanagerArgs) - } - - if versionutil.MustParseSemantic(version).LessThan(versionutil.MustParseSemantic("1.19.0")) { - delete(args, "cluster-signing-duration") - args["experimental-cluster-signing-duration"] = "87600h" - } - return args -} - -func GetSchedulerArgs(securityEnhancement bool) map[string]string { - if securityEnhancement { - return SchedulerSecurityArgs - } - return SchedulerArgs -} - -func UpdateFeatureGatesConfiguration(args map[string]string, kubeConf *common.KubeConf) map[string]string { - var featureGates []string - - for k, v := range kubeConf.Cluster.Kubernetes.FeatureGates { - featureGates = append(featureGates, fmt.Sprintf("%s=%v", k, v)) - } - - for k, v := range FeatureGatesDefaultConfiguration { - // When kubernetes version is less than 1.21,`CSIStorageCapacity` should not be set. - if k == "CSIStorageCapacity" && - versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).LessThan(versionutil.MustParseSemantic("v1.21.0")) { - continue - } - if k == "TTLAfterFinished" && - versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.24.0")) { - continue - } - - if _, ok := kubeConf.Cluster.Kubernetes.FeatureGates[k]; !ok { - featureGates = append(featureGates, fmt.Sprintf("%s=%v", k, v)) - } - } - - args["feature-gates"] = strings.Join(featureGates, ",") - - return args -} - -func GetKubeletConfiguration(runtime connector.Runtime, kubeConf *common.KubeConf, criSock string, securityEnhancement bool) map[string]interface{} { - // When kubernetes version is less than 1.21,`CSIStorageCapacity` should not be set. - cmp, _ := versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).Compare("v1.21.0") - if cmp == -1 { - delete(FeatureGatesDefaultConfiguration, "CSIStorageCapacity") - } - - defaultKubeletConfiguration := map[string]interface{}{ - "clusterDomain": kubeConf.Cluster.Kubernetes.DNSDomain, - "clusterDNS": []string{kubeConf.Cluster.ClusterDNS()}, - "maxPods": kubeConf.Cluster.Kubernetes.MaxPods, - "podPidsLimit": kubeConf.Cluster.Kubernetes.PodPidsLimit, - "rotateCertificates": true, - "kubeReserved": map[string]string{ - "cpu": "200m", - "memory": "250Mi", - }, - "systemReserved": map[string]string{ - "cpu": "200m", - "memory": "250Mi", - }, - "evictionHard": map[string]string{ - "memory.available": "5%", - "pid.available": "10%", - }, - "evictionSoft": map[string]string{ - "memory.available": "10%", - }, - "evictionSoftGracePeriod": map[string]string{ - "memory.available": "2m", - }, - "evictionMaxPodGracePeriod": 120, - "evictionPressureTransitionPeriod": "30s", - "featureGates": FeatureGatesDefaultConfiguration, - } - - if securityEnhancement { - defaultKubeletConfiguration["readOnlyPort"] = 0 - defaultKubeletConfiguration["protectKernelDefaults"] = true - defaultKubeletConfiguration["eventRecordQPS"] = 1 - defaultKubeletConfiguration["streamingConnectionIdleTimeout"] = "5m" - defaultKubeletConfiguration["makeIPTablesUtilChains"] = true - defaultKubeletConfiguration["tlsCipherSuites"] = []string{ - "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", - "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", - } - defaultKubeletConfiguration["featureGates"] = FeatureGatesSecurityDefaultConfiguration - } - - cgroupDriver, err := GetKubeletCgroupDriver(runtime, kubeConf) - if err != nil { - logger.Log.Fatal(err) - } - if len(cgroupDriver) == 0 { - defaultKubeletConfiguration["cgroupDriver"] = "systemd" - } - - if len(criSock) != 0 { - defaultKubeletConfiguration["containerLogMaxSize"] = "5Mi" - defaultKubeletConfiguration["containerLogMaxFiles"] = 3 - } - - customKubeletConfiguration := make(map[string]interface{}) - if len(kubeConf.Cluster.Kubernetes.KubeletConfiguration.Raw) != 0 { - err := yaml.Unmarshal(kubeConf.Cluster.Kubernetes.KubeletConfiguration.Raw, &customKubeletConfiguration) - if err != nil { - logger.Log.Fatal("failed to parse kubelet configuration") - } - } - - kubeletConfiguration := make(map[string]interface{}) - if len(customKubeletConfiguration) != 0 { - for customArg := range customKubeletConfiguration { - if _, ok := defaultKubeletConfiguration[customArg]; ok { - kubeletConfiguration[customArg] = customKubeletConfiguration[customArg] - delete(defaultKubeletConfiguration, customArg) - delete(customKubeletConfiguration, customArg) - } else { - kubeletConfiguration[customArg] = customKubeletConfiguration[customArg] - } - } - } - - if len(defaultKubeletConfiguration) != 0 { - for k, v := range defaultKubeletConfiguration { - kubeletConfiguration[k] = v - } - } - - if featureGates, ok := kubeletConfiguration["featureGates"].(map[string]bool); ok { - if versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).LessThan(versionutil.MustParseSemantic("v1.21.0")) { - delete(featureGates, "CSIStorageCapacity") - } - - if versionutil.MustParseSemantic(kubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.24.0")) { - delete(featureGates, "TTLAfterFinished") - } - - for k, v := range kubeConf.Cluster.Kubernetes.FeatureGates { - if _, ok := featureGates[k]; !ok { - featureGates[k] = v - } - } - } - - if kubeConf.Arg.Debug { - logger.Log.Debugf("Set kubeletConfiguration: %v", kubeletConfiguration) - } - - return kubeletConfiguration -} - -func GetKubeletCgroupDriver(runtime connector.Runtime, kubeConf *common.KubeConf) (string, error) { - var cmd, kubeletCgroupDriver string - switch kubeConf.Cluster.Kubernetes.ContainerManager { - case common.Docker, "": - cmd = "docker info | grep 'Cgroup Driver'" - case common.Crio: - cmd = "crio config | grep cgroup_manager" - case common.Containerd: - cmd = "containerd config dump | grep SystemdCgroup || echo 'SystemdCgroup = false'" - case common.Isula: - cmd = "isula info | grep 'Cgroup Driver'" - default: - kubeletCgroupDriver = "" - } - - checkResult, err := runtime.GetRunner().SudoCmd(cmd, false) - if err != nil { - return "", errors.Wrap(errors.WithStack(err), "Failed to get container runtime cgroup driver.") - } - if strings.Contains(checkResult, "systemd") || strings.Contains(checkResult, "SystemdCgroup = true") { - kubeletCgroupDriver = "systemd" - } else if strings.Contains(checkResult, "cgroupfs") || strings.Contains(checkResult, "SystemdCgroup = false") { - kubeletCgroupDriver = "cgroupfs" - } else { - return "", errors.Errorf("Failed to get container runtime cgroup driver from %s by run %s", checkResult, cmd) - } - return kubeletCgroupDriver, nil -} - -func GetKubeProxyConfiguration(kubeConf *common.KubeConf) map[string]interface{} { - defaultKubeProxyConfiguration := map[string]interface{}{ - "clusterCIDR": kubeConf.Cluster.Network.KubePodsCIDR, - "mode": kubeConf.Cluster.Kubernetes.ProxyMode, - "iptables": map[string]interface{}{ - "masqueradeAll": kubeConf.Cluster.Kubernetes.MasqueradeAll, - "masqueradeBit": 14, - "minSyncPeriod": "0s", - "syncPeriod": "30s", - }, - } - - customKubeProxyConfiguration := make(map[string]interface{}) - if len(kubeConf.Cluster.Kubernetes.KubeProxyConfiguration.Raw) != 0 { - err := yaml.Unmarshal(kubeConf.Cluster.Kubernetes.KubeProxyConfiguration.Raw, &customKubeProxyConfiguration) - if err != nil { - logger.Log.Fatal("failed to parse kube-proxy's configuration") - } - } - - kubeProxyConfiguration := make(map[string]interface{}) - if len(customKubeProxyConfiguration) != 0 { - for customArg := range customKubeProxyConfiguration { - if _, ok := defaultKubeProxyConfiguration[customArg]; ok { - kubeProxyConfiguration[customArg] = customKubeProxyConfiguration[customArg] - delete(defaultKubeProxyConfiguration, customArg) - delete(customKubeProxyConfiguration, customArg) - } else { - kubeProxyConfiguration[customArg] = customKubeProxyConfiguration[customArg] - } - } - } - - if len(defaultKubeProxyConfiguration) != 0 { - for defaultArg := range defaultKubeProxyConfiguration { - kubeProxyConfiguration[defaultArg] = defaultKubeProxyConfiguration[defaultArg] - } - } - - return kubeProxyConfiguration -} - -func copyStringMap(m map[string]string) map[string]string { - cp := make(map[string]string) - for k, v := range m { - cp[k] = v - } - - return cp -} diff --git a/cmd/kk/pkg/kubernetes/templates/kubelet_env.go b/cmd/kk/pkg/kubernetes/templates/kubelet_env.go deleted file mode 100644 index e5170eff..00000000 --- a/cmd/kk/pkg/kubernetes/templates/kubelet_env.go +++ /dev/null @@ -1,38 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -// KubeletEnv defines the template of kubelet's Env for the kubelet's systemd service. -var KubeletEnv = template.Must(template.New("10-kubeadm.conf").Parse( - dedent.Dedent(`# Note: This dropin only works with kubeadm and kubelet v1.11+ -[Service] -Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" -Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" -# This is a file that "kubeadm init" and "kubeadm join" generate at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically -EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env -# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use -# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file. -EnvironmentFile=-/etc/default/kubelet -Environment="KUBELET_EXTRA_ARGS=--node-ip={{ .NodeIP }} --hostname-override={{ .Hostname }} {{ if .ContainerRuntime }}--network-plugin=cni{{ end }} {{range .KubeletArgs }} {{.}}{{ end }}" -ExecStart= -ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS - `))) diff --git a/cmd/kk/pkg/kubernetes/templates/kubelet_service.go b/cmd/kk/pkg/kubernetes/templates/kubelet_service.go deleted file mode 100644 index d4752f67..00000000 --- a/cmd/kk/pkg/kubernetes/templates/kubelet_service.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -// KubeletService defines the template of kubelete service for systemd. -var KubeletService = template.Must(template.New("kubelet.service").Parse( - dedent.Dedent(`[Unit] -Description=kubelet: The Kubernetes Node Agent -Documentation=http://kubernetes.io/docs/ - -[Service] -CPUAccounting=true -MemoryAccounting=true -ExecStart=/usr/local/bin/kubelet -Restart=always -StartLimitInterval=0 -RestartSec=10 - -[Install] -WantedBy=multi-user.target - `))) diff --git a/cmd/kk/pkg/kubernetes/upgrade_step_enum.go b/cmd/kk/pkg/kubernetes/upgrade_step_enum.go deleted file mode 100644 index 037b80ba..00000000 --- a/cmd/kk/pkg/kubernetes/upgrade_step_enum.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -type UpgradeStep int - -const ( - ToV121 UpgradeStep = iota + 1 - ToV122 -) - -var UpgradeStepList = []UpgradeStep{ - ToV121, - ToV122, -} - -func (u UpgradeStep) String() string { - switch u { - case ToV121: - return "to v1.21" - case ToV122: - return "to v1.22" - default: - return "invalid option" - } -} diff --git a/cmd/kk/pkg/kubesphere/kubesphere_test.go b/cmd/kk/pkg/kubesphere/kubesphere_test.go deleted file mode 100644 index 46a63918..00000000 --- a/cmd/kk/pkg/kubesphere/kubesphere_test.go +++ /dev/null @@ -1,99 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubesphere - -import ( - "reflect" - "testing" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -func Test_mirrorRepo(t *testing.T) { - tests := []struct { - name string - version string - want string - }{ - { - name: "test_latest", - version: "latest", - want: "kubespheredev", - }, - { - name: "test_master", - version: "master", - want: "kubespheredev", - }, - { - name: "test_v3.2.1-rc.1", - version: "v3.2.1-rc.1", - want: "kubespheredev", - }, - { - name: "test_v3.2.1", - version: "v3.2.1", - want: "kubesphere", - }, - { - name: "test_v3.2.0", - version: "v3.2.0", - want: "kubesphere", - }, - { - name: "test_3.2.0", - version: "3.2.0", - want: "kubespheredev", - }, - { - name: "test_v3.2.0-alpha.1", - version: "v3.2.0-alpha.1", - want: "kubespheredev", - }, - { - name: "test_v1.2.0", - version: "v1.2.0", - want: "kubesphere", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := repo(tt.version) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("StabledVersionSupport() got = %v, want %v", got, tt.want) - } - }) - } -} - -func repo(version string) string { - var r string - _, latest := kubesphere.LatestRelease(version) - _, dev := kubesphere.DevRelease(version) - _, stable := kubesphere.StabledVersionSupport(version) - switch { - case stable: - r = "kubesphere" - case dev: - r = "kubespheredev" - case latest: - r = "kubespheredev" - default: - r = "kubesphere" - } - return r -} diff --git a/cmd/kk/pkg/kubesphere/modules.go b/cmd/kk/pkg/kubesphere/modules.go deleted file mode 100644 index f7628f3c..00000000 --- a/cmd/kk/pkg/kubesphere/modules.go +++ /dev/null @@ -1,246 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubesphere - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere/templates" -) - -type DeployModule struct { - common.KubeModule - Skip bool -} - -func (d *DeployModule) IsSkip() bool { - return d.Skip -} - -func (d *DeployModule) Init() { - d.Name = "DeployKubeSphereModule" - d.Desc = "Deploy KubeSphere" - - generateManifests := &task.RemoteTask{ - Name: "GenerateKsInstallerCRD", - Desc: "Generate KubeSphere ks-installer crd manifests", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(NotEqualDesiredVersion), - }, - Action: &action.Template{ - Template: templates.KsInstaller, - Dst: filepath.Join(common.KubeAddonsDir, templates.KsInstaller.Name()), - Data: util.Data{ - "Repo": MirrorRepo(d.KubeConf), - "Tag": d.KubeConf.Cluster.KubeSphere.Version, - }, - }, - Parallel: true, - } - - addConfig := &task.RemoteTask{ - Name: "AddKsInstallerConfig", - Desc: "Add config to ks-installer manifests", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(NotEqualDesiredVersion), - }, - Action: new(AddInstallerConfig), - Parallel: true, - } - - createNamespace := &task.RemoteTask{ - Name: "CreateKubeSphereNamespace", - Desc: "Create the kubesphere namespace", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(NotEqualDesiredVersion), - }, - Action: new(CreateNamespace), - Parallel: true, - } - - setup := &task.RemoteTask{ - Name: "SetupKsInstallerConfig", - Desc: "Setup ks-installer config", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(NotEqualDesiredVersion), - }, - Action: new(Setup), - Parallel: true, - } - - apply := &task.RemoteTask{ - Name: "ApplyKsInstaller", - Desc: "Apply ks-installer", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(NotEqualDesiredVersion), - }, - Action: new(Apply), - Parallel: true, - } - - d.Tasks = []task.Interface{ - generateManifests, - // apply crd installer.kubesphere.io/v1alpha1 - apply, - addConfig, - createNamespace, - setup, - apply, - } -} - -func MirrorRepo(kubeConf *common.KubeConf) string { - repo := kubeConf.Cluster.Registry.PrivateRegistry - namespaceOverride := kubeConf.Cluster.Registry.NamespaceOverride - version := kubeConf.Cluster.KubeSphere.Version - - _, ok := kubesphere.CNSource[version] - if ok && os.Getenv("KKZONE") == "cn" { - if repo == "" { - repo = "registry.cn-beijing.aliyuncs.com/kubesphereio" - } else if len(namespaceOverride) != 0 { - repo = fmt.Sprintf("%s/%s", repo, namespaceOverride) - } else { - repo = fmt.Sprintf("%s/kubesphere", repo) - } - } else { - if repo == "" { - _, latest := kubesphere.LatestRelease(version) - _, dev := kubesphere.DevRelease(version) - _, stable := kubesphere.StabledVersionSupport(version) - switch { - case stable: - repo = "kubesphere" - case dev: - repo = "kubespheredev" - case latest: - repo = "kubespheredev" - default: - repo = "kubesphere" - } - } else if len(namespaceOverride) != 0 { - repo = fmt.Sprintf("%s/%s", repo, namespaceOverride) - } else { - repo = fmt.Sprintf("%s/kubesphere", repo) - } - } - return repo -} - -type CheckResultModule struct { - common.KubeModule - Skip bool -} - -func (c *CheckResultModule) IsSkip() bool { - return c.Skip -} - -func (c *CheckResultModule) Init() { - c.Name = "CheckResultModule" - c.Desc = "Check deploy KubeSphere result" - - check := &task.RemoteTask{ - Name: "CheckKsInstallerResult", - Desc: "Check ks-installer result", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(NotEqualDesiredVersion), - }, - Action: new(Check), - Parallel: true, - } - - c.Tasks = []task.Interface{ - check, - } -} - -type CleanClusterConfigurationModule struct { - common.KubeModule - Skip bool -} - -func (c *CleanClusterConfigurationModule) IsSkip() bool { - return c.Skip -} - -func (c *CleanClusterConfigurationModule) Init() { - c.Name = "CleanClusterConfigurationModule" - c.Desc = "Clean redundant ClusterConfiguration config" - - // ensure there is no cc config, and prevent to reset cc config when upgrade the cluster - clean := &task.LocalTask{ - Name: "CleanClusterConfiguration", - Desc: "Clean redundant ClusterConfiguration config", - Action: new(CleanCC), - } - - c.Tasks = []task.Interface{ - clean, - } -} - -type ConvertModule struct { - common.KubeModule - Skip bool -} - -func (c *ConvertModule) IsSkip() bool { - return c.Skip -} - -func (c *ConvertModule) Init() { - c.Name = "ConvertModule" - c.Desc = "Convert ks-installer config v2 to v3" - - convert := &task.RemoteTask{ - Name: "ConvertV2ToV3", - Desc: "Convert ks-installer config v2 to v3", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(NotEqualDesiredVersion), - new(VersionBelowV3), - }, - Action: new(ConvertV2ToV3), - Parallel: true, - } - - c.Tasks = []task.Interface{ - convert, - } -} diff --git a/cmd/kk/pkg/kubesphere/prepares.go b/cmd/kk/pkg/kubesphere/prepares.go deleted file mode 100644 index 44dc28b1..00000000 --- a/cmd/kk/pkg/kubesphere/prepares.go +++ /dev/null @@ -1,58 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubesphere - -import ( - "github.com/pkg/errors" - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type VersionBelowV3 struct { - common.KubePrepare -} - -func (v *VersionBelowV3) PreCheck(runtime connector.Runtime) (bool, error) { - versionStr, ok := v.PipelineCache.GetMustString(common.KubeSphereVersion) - if !ok { - return false, errors.New("get current kubesphere version failed by pipeline cache") - } - version := versionutil.MustParseSemantic(versionStr) - v300 := versionutil.MustParseSemantic("v3.0.0") - if v.KubeConf.Cluster.KubeSphere.Enabled && v.KubeConf.Cluster.KubeSphere.Version == "v3.0.0" && version.LessThan(v300) { - return true, nil - } - return false, nil -} - -type NotEqualDesiredVersion struct { - common.KubePrepare -} - -func (n *NotEqualDesiredVersion) PreCheck(runtime connector.Runtime) (bool, error) { - ksVersion, ok := n.PipelineCache.GetMustString(common.KubeSphereVersion) - if !ok { - ksVersion = "" - } - - if n.KubeConf.Cluster.KubeSphere.Version == ksVersion { - return false, nil - } - return true, nil -} diff --git a/cmd/kk/pkg/kubesphere/tasks.go b/cmd/kk/pkg/kubesphere/tasks.go deleted file mode 100644 index e867fb4e..00000000 --- a/cmd/kk/pkg/kubesphere/tasks.go +++ /dev/null @@ -1,433 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubesphere - -import ( - "context" - "encoding/base64" - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "github.com/pkg/errors" - yamlV3 "gopkg.in/yaml.v3" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - ksv2 "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubesphere/v2" - ksv3 "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubesphere/v3" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere/templates" -) - -type AddInstallerConfig struct { - common.KubeAction -} - -func (a *AddInstallerConfig) Execute(runtime connector.Runtime) error { - configurationBase64 := base64.StdEncoding.EncodeToString([]byte(a.KubeConf.Cluster.KubeSphere.Configurations)) - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("echo %s | base64 -d >> /etc/kubernetes/addons/kubesphere.yaml", configurationBase64), - false); err != nil { - return errors.Wrap(errors.WithStack(err), "add config to ks-installer manifests failed") - } - return nil -} - -type CreateNamespace struct { - common.KubeAction -} - -func (c *CreateNamespace) Execute(runtime connector.Runtime) error { - _, err := runtime.GetRunner().SudoCmd(`cat <>--->", - ) - - fmt.Printf("%s \033[K\n", output) - time.Sleep(time.Duration(200) * time.Millisecond) - } else { - fmt.Printf("\033[%dA\033[K", position) - - output := fmt.Sprintf( - "%s%s%s", - notes, - strings.Repeat(" ", 10-i), - "<---<<", - ) - - fmt.Printf("%s \033[K\n", output) - time.Sleep(time.Duration(200) * time.Millisecond) - } - } - } - } - return nil -} - -func CheckKubeSphereStatus(ctx context.Context, runtime connector.Runtime, stopChan chan string) { - defer close(stopChan) - for { - select { - case <-ctx.Done(): - stopChan <- "" - default: - _, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl exec -n kubesphere-system "+ - "$(kubectl get pod -n kubesphere-system -l app=ks-installer -o jsonpath='{.items[0].metadata.name}') "+ - "-- ls /kubesphere/playbooks/kubesphere_running", false) - if err == nil { - output, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl exec -n kubesphere-system "+ - "$(kubectl get pod -n kubesphere-system -l app=ks-installer -o jsonpath='{.items[0].metadata.name}') "+ - "-- cat /kubesphere/playbooks/kubesphere_running", false) - if err == nil && output != "" { - stopChan <- output - break - } - } - } - } -} - -type CleanCC struct { - common.KubeAction -} - -func (c *CleanCC) Execute(runtime connector.Runtime) error { - c.KubeConf.Cluster.KubeSphere.Configurations = "\n" - return nil -} - -type ConvertV2ToV3 struct { - common.KubeAction -} - -func (c *ConvertV2ToV3) Execute(runtime connector.Runtime) error { - configV2Str, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl get cm -n kubesphere-system ks-installer -o jsonpath='{.data.ks-config\\.yaml}'", - false) - if err != nil { - return err - } - - clusterCfgV2 := ksv2.V2{} - clusterCfgV3 := ksv3.V3{} - if err := yamlV3.Unmarshal([]byte(configV2Str), &clusterCfgV2); err != nil { - return err - } - - configV3, err := MigrateConfig2to3(&clusterCfgV2, &clusterCfgV3) - if err != nil { - return err - } - c.KubeConf.Cluster.KubeSphere.Configurations = "---\n" + configV3 - return nil -} - -func MigrateConfig2to3(v2 *ksv2.V2, v3 *ksv3.V3) (string, error) { - v3.Etcd = ksv3.Etcd(v2.Etcd) - v3.Persistence = ksv3.Persistence(v2.Persistence) - v3.Alerting = ksv3.Alerting(v2.Alerting) - v3.Notification = ksv3.Notification(v2.Notification) - v3.LocalRegistry = v2.LocalRegistry - v3.Servicemesh = ksv3.Servicemesh(v2.Servicemesh) - v3.Devops = ksv3.Devops(v2.Devops) - v3.Openpitrix = ksv3.Openpitrix(v2.Openpitrix) - v3.Console = ksv3.Console(v2.Console) - - if v2.MetricsServerNew.Enabled == "" { - if v2.MetricsServerOld.Enabled == "true" || v2.MetricsServerOld.Enabled == "True" { - v3.MetricsServer.Enabled = true - } else { - v3.MetricsServer.Enabled = false - } - } else { - if v2.MetricsServerNew.Enabled == "true" || v2.MetricsServerNew.Enabled == "True" { - v3.MetricsServer.Enabled = true - } else { - v3.MetricsServer.Enabled = false - } - } - - v3.Monitoring.PrometheusMemoryRequest = v2.Monitoring.PrometheusMemoryRequest - //v3.Monitoring.PrometheusReplicas = v2.Monitoring.PrometheusReplicas - v3.Monitoring.PrometheusVolumeSize = v2.Monitoring.PrometheusVolumeSize - //v3.Monitoring.AlertmanagerReplicas = 1 - - v3.Common.EtcdVolumeSize = v2.Common.EtcdVolumeSize - v3.Common.MinioVolumeSize = v2.Common.MinioVolumeSize - v3.Common.MysqlVolumeSize = v2.Common.MysqlVolumeSize - v3.Common.OpenldapVolumeSize = v2.Common.OpenldapVolumeSize - v3.Common.RedisVolumSize = v2.Common.RedisVolumSize - //v3.Common.ES.ElasticsearchDataReplicas = v2.Logging.ElasticsearchDataReplicas - //v3.Common.ES.ElasticsearchMasterReplicas = v2.Logging.ElasticsearchMasterReplicas - v3.Common.ES.ElkPrefix = v2.Logging.ElkPrefix - v3.Common.ES.LogMaxAge = v2.Logging.LogMaxAge - if v2.Logging.ElasticsearchVolumeSize == "" { - v3.Common.ES.ElasticsearchDataVolumeSize = v2.Logging.ElasticsearchDataVolumeSize - v3.Common.ES.ElasticsearchMasterVolumeSize = v2.Logging.ElasticsearchMasterVolumeSize - } else { - v3.Common.ES.ElasticsearchMasterVolumeSize = "4Gi" - v3.Common.ES.ElasticsearchDataVolumeSize = v2.Logging.ElasticsearchVolumeSize - } - - v3.Logging.Enabled = v2.Logging.Enabled - v3.Logging.LogsidecarReplicas = v2.Logging.LogsidecarReplicas - - v3.Authentication.JwtSecret = "" - v3.Multicluster.ClusterRole = "none" - v3.Events.Ruler.Replicas = 2 - - var clusterConfiguration = ksv3.ClusterConfig{ - ApiVersion: "installer.kubesphere.io/v1alpha1", - Kind: "ClusterConfiguration", - Metadata: ksv3.Metadata{ - Name: "ks-installer", - Namespace: "kubesphere-system", - Label: ksv3.Label{Version: "v3.0.0"}, - }, - Spec: v3, - } - - configV3, err := yamlV3.Marshal(clusterConfiguration) - if err != nil { - return "", err - } - - return string(configV3), nil -} diff --git a/cmd/kk/pkg/kubesphere/v2/types.go b/cmd/kk/pkg/kubesphere/v2/types.go deleted file mode 100644 index 9d3ce32d..00000000 --- a/cmd/kk/pkg/kubesphere/v2/types.go +++ /dev/null @@ -1,109 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v2 - -type V2 struct { - Persistence Persistence `yaml:"persistence"` - Common Common `yaml:"common"` - Etcd Etcd `yaml:"etcd"` - MetricsServerOld MetricsServerOld `yaml:"metrics-server"` - MetricsServerNew MetricsServerNew `yaml:"metrics_server"` - Console Console `yaml:"console"` - Monitoring Monitoring `yaml:"monitoring"` - Logging Logging `yaml:"logging"` - Openpitrix Openpitrix `yaml:"openpitrix"` - Devops Devops `yaml:"devops"` - Servicemesh Servicemesh `yaml:"servicemesh"` - Notification Notification `yaml:"notification"` - Alerting Alerting `yaml:"alerting"` - LocalRegistry string `yaml:"local_registry"` -} - -type Persistence struct { - StorageClass string `yaml:"storageClass"` -} -type Etcd struct { - Monitoring bool `yaml:"monitoring"` - EndpointIps string `yaml:"endpointIps"` - Port int `yaml:"port"` - TlsEnable bool `yaml:"tlsEnable"` -} - -type Common struct { - MysqlVolumeSize string `yaml:"mysqlVolumeSize"` - MinioVolumeSize string `yaml:"minioVolumeSize"` - EtcdVolumeSize string `yaml:"etcdVolumeSize"` - OpenldapVolumeSize string `yaml:"openldapVolumeSize"` - RedisVolumSize string `yaml:"redisVolumSize"` -} - -type MetricsServerOld struct { - Enabled string `yaml:"enabled"` -} - -type MetricsServerNew struct { - Enabled string `yaml:"enabled"` -} - -type Console struct { - EnableMultiLogin bool `yaml:"enableMultiLogin"` - Port int `yaml:"port"` -} - -type Monitoring struct { - PrometheusReplicas int `yaml:"prometheusReplicas"` - PrometheusMemoryRequest string `yaml:"prometheusMemoryRequest"` - PrometheusVolumeSize string `yaml:"prometheusVolumeSize"` -} - -type Logging struct { - Enabled bool `yaml:"enabled"` - ElasticsearchMasterReplicas int `yaml:"elasticsearchMasterReplicas"` - ElasticsearchDataReplicas int `yaml:"elasticsearchDataReplicas"` - LogsidecarReplicas int `yaml:"logsidecarReplicas"` - ElasticsearchVolumeSize string `yaml:"elasticsearchVolumeSize"` - ElasticsearchMasterVolumeSize string `yaml:"elasticsearchMasterVolumeSize"` - ElasticsearchDataVolumeSize string `yaml:"elasticsearchDataVolumeSize"` - LogMaxAge int `yaml:"logMaxAge"` - ElkPrefix string `yaml:"elkPrefix"` -} - -type Openpitrix struct { - Enabled bool `yaml:"enabled"` -} - -type Devops struct { - Enabled bool `yaml:"enabled"` - JenkinsMemoryLim string `yaml:"jenkinsMemoryLim"` - JenkinsMemoryReq string `yaml:"jenkinsMemoryReq"` - JenkinsVolumeSize string `yaml:"jenkinsVolumeSize"` - JenkinsjavaoptsXms string `yaml:"jenkinsJavaOpts_Xms"` - JenkinsjavaoptsXmx string `yaml:"jenkinsJavaOpts_Xmx"` - JenkinsjavaoptsMaxram string `yaml:"jenkinsJavaOpts_MaxRAM"` -} - -type Servicemesh struct { - Enabled bool `yaml:"enabled"` -} - -type Notification struct { - Enabled bool `yaml:"enabled"` -} - -type Alerting struct { - Enabled bool `yaml:"enabled"` -} diff --git a/cmd/kk/pkg/kubesphere/v3/types.go b/cmd/kk/pkg/kubesphere/v3/types.go deleted file mode 100644 index 97b97198..00000000 --- a/cmd/kk/pkg/kubesphere/v3/types.go +++ /dev/null @@ -1,160 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v3 - -type ClusterConfig struct { - ApiVersion string `yaml:"apiVersion"` - Kind string `yaml:"kind"` - Metadata Metadata `yaml:"metadata"` - Spec *V3 `yaml:"spec"` -} - -type Metadata struct { - Name string `yaml:"name"` - Namespace string `yaml:"namespace"` - Label Label `yaml:"labels"` -} - -type Label struct { - Version string `yaml:"version"` -} - -type V3 struct { - Persistence Persistence `yaml:"persistence"` - Authentication Authentication `yaml:"authentication"` - Common Common `yaml:"common"` - Etcd Etcd `yaml:"etcd"` - MetricsServer MetricsServer `yaml:"metrics_server"` - Console Console `yaml:"console"` - Monitoring Monitoring `yaml:"monitoring"` - Logging Logging `yaml:"logging"` - Openpitrix Openpitrix `yaml:"openpitrix"` - Devops Devops `yaml:"devops"` - Servicemesh Servicemesh `yaml:"servicemesh"` - Notification Notification `yaml:"notification"` - Alerting Alerting `yaml:"alerting"` - Auditing Auditing `yaml:"auditing"` - Events Events `yaml:"events"` - Multicluster Multicluster `yaml:"multicluster"` - Networkpolicy Networkpolicy `yaml:"networkpolicy"` - LocalRegistry string `yaml:"local_registry"` -} - -type Persistence struct { - StorageClass string `yaml:"storageClass"` -} - -type MetricsServer struct { - Enabled bool `yaml:"enabled"` -} - -type Authentication struct { - JwtSecret string `yaml:"jwtSecret"` -} -type Etcd struct { - Monitoring bool `yaml:"monitoring"` - EndpointIps string `yaml:"endpointIps"` - Port int `yaml:"port"` - TlsEnable bool `yaml:"tlsEnable"` -} - -type Common struct { - MysqlVolumeSize string `yaml:"mysqlVolumeSize"` - MinioVolumeSize string `yaml:"minioVolumeSize"` - EtcdVolumeSize string `yaml:"etcdVolumeSize"` - OpenldapVolumeSize string `yaml:"openldapVolumeSize"` - RedisVolumSize string `yaml:"redisVolumSize"` - ES ES `yaml:"es"` -} - -type ES struct { - //ElasticsearchMasterReplicas int `yaml:"elasticsearchMasterReplicas"` - //ElasticsearchDataReplicas int `yaml:"elasticsearchDataReplicas"` - ElasticsearchMasterVolumeSize string `yaml:"elasticsearchMasterVolumeSize"` - ElasticsearchDataVolumeSize string `yaml:"elasticsearchDataVolumeSize"` - LogMaxAge int `yaml:"logMaxAge"` - ElkPrefix string `yaml:"elkPrefix"` -} - -type Console struct { - EnableMultiLogin bool `yaml:"enableMultiLogin"` - Port int `yaml:"port"` -} - -type Alerting struct { - Enabled bool `yaml:"enabled"` -} - -type Auditing struct { - Enabled bool `yaml:"enabled"` -} - -type Devops struct { - Enabled bool `yaml:"enabled"` - JenkinsMemoryLim string `yaml:"jenkinsMemoryLim"` - JenkinsMemoryReq string `yaml:"jenkinsMemoryReq"` - JenkinsVolumeSize string `yaml:"jenkinsVolumeSize"` - JenkinsjavaoptsXms string `yaml:"jenkinsJavaOpts_Xms"` - JenkinsjavaoptsXmx string `yaml:"jenkinsJavaOpts_Xmx"` - JenkinsjavaoptsMaxram string `yaml:"jenkinsJavaOpts_MaxRAM"` -} - -type Events struct { - Enabled bool `yaml:"enabled"` - Ruler Ruler `yaml:"ruler"` -} - -type Ruler struct { - Enabled bool `yaml:"enabled"` - Replicas int `yaml:"replicas"` -} - -type Logging struct { - Enabled bool `yaml:"enabled"` - LogsidecarReplicas int `yaml:"logsidecarReplicas"` -} - -type Metrics struct { - Enabled bool `yaml:"enabled"` -} - -type Monitoring struct { - //AlertmanagerReplicas int `yaml:"alertmanagerReplicas"` - //PrometheusReplicas int `yaml:"prometheusReplicas"` - PrometheusMemoryRequest string `yaml:"prometheusMemoryRequest"` - PrometheusVolumeSize string `yaml:"prometheusVolumeSize"` -} - -type Multicluster struct { - ClusterRole string `yaml:"clusterRole"` -} - -type Networkpolicy struct { - Enabled bool `yaml:"enabled"` -} - -type Notification struct { - Enabled bool `yaml:"enabled"` -} - -type Openpitrix struct { - Enabled bool `yaml:"enabled"` -} - -type Servicemesh struct { - Enabled bool `yaml:"enabled"` -} diff --git a/cmd/kk/pkg/loadbalancer/common.go b/cmd/kk/pkg/loadbalancer/common.go deleted file mode 100644 index d53b16fc..00000000 --- a/cmd/kk/pkg/loadbalancer/common.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package loadbalancer - -const ( - LocalServer = "server: https://127.0.0.1" -) diff --git a/cmd/kk/pkg/loadbalancer/module.go b/cmd/kk/pkg/loadbalancer/module.go deleted file mode 100644 index 7492581f..00000000 --- a/cmd/kk/pkg/loadbalancer/module.go +++ /dev/null @@ -1,380 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package loadbalancer - -import ( - "path/filepath" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/loadbalancer/templates" -) - -type HaproxyModule struct { - common.KubeModule - Skip bool -} - -func (h *HaproxyModule) IsSkip() bool { - return h.Skip -} - -func (h *HaproxyModule) Init() { - h.Name = "InternalLoadbalancerModule" - h.Desc = "Install internal load balancer" - - haproxyCfg := &task.RemoteTask{ - Name: "GenerateHaproxyConfig", - Desc: "Generate haproxy.cfg", - Hosts: h.Runtime.GetHostsByRole(common.Worker), - Prepare: new(common.OnlyWorker), - Action: &action.Template{ - Template: templates.HaproxyConfig, - Dst: filepath.Join(common.HaproxyDir, templates.HaproxyConfig.Name()), - Data: util.Data{ - "MasterNodes": templates.MasterNodeStr(h.Runtime, h.KubeConf), - "LoadbalancerApiserverPort": kubekeyapiv1alpha2.DefaultApiserverPort, - "LoadbalancerApiserverHealthcheckPort": 8081, - "KubernetesType": h.KubeConf.Cluster.Kubernetes.Type, - }, - }, - Parallel: true, - } - - // Calculation config md5 as the checksum. - // It will make load balancer reload when config changes. - getMd5Sum := &task.RemoteTask{ - Name: "GetChecksumFromConfig", - Desc: "Calculate the MD5 value according to haproxy.cfg", - Hosts: h.Runtime.GetHostsByRole(common.Worker), - Prepare: new(common.OnlyWorker), - Action: new(GetChecksum), - Parallel: true, - } - - haproxyManifestK8s := &task.RemoteTask{ - Name: "GenerateHaproxyManifest", - Desc: "Generate haproxy manifest", - Hosts: h.Runtime.GetHostsByRole(common.Worker), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyWorker), - new(common.OnlyKubernetes), - }, - Action: new(GenerateHaproxyManifest), - Parallel: true, - } - - // UpdateKubeletConfig Update server field in kubelet.conf - // When create a HA cluster by internal LB, we will set the server filed to 127.0.0.1:6443 (default) which in kubelet.conf. - // Because of that, the control plone node's kubelet connect the local api-server. - // And the work node's kubelet connect 127.0.0.1:6443 (default) that is proxy by the node's local nginx. - updateKubeletConfig := &task.RemoteTask{ - Name: "UpdateKubeletConfig", - Desc: "Update kubelet config", - Hosts: h.Runtime.GetHostsByRole(common.K8s), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyKubernetes), - new(updateKubeletPrepare), - }, - Action: new(UpdateKubelet), - Parallel: true, - Retry: 3, - } - - // updateKubeProxyConfig is used to update kube-proxy configmap and restart tge kube-proxy pod. - updateKubeProxyConfig := &task.RemoteTask{ - Name: "UpdateKubeProxyConfig", - Desc: "Update kube-proxy configmap", - Hosts: []connector.Host{h.Runtime.GetHostsByRole(common.Master)[0]}, - Prepare: &prepare.PrepareCollection{ - new(common.EnableKubeProxy), - new(common.OnlyKubernetes), - new(common.OnlyFirstMaster), - new(updateKubeProxyPrapre), - }, - Action: new(UpdateKubeProxy), - Parallel: true, - Retry: 3, - } - - // UpdateHostsFile is used to update the '/etc/hosts'. Make the 'lb.kubesphere.local' address to set as 127.0.0.1. - // All of the 'admin.conf' and '/.kube/config' will connect to 127.0.0.1:6443. - updateHostsFile := &task.RemoteTask{ - Name: "UpdateHostsFile", - Desc: "Update /etc/hosts", - Hosts: h.Runtime.GetHostsByRole(common.K8s), - Action: new(UpdateHosts), - Parallel: true, - Retry: 3, - } - - h.Tasks = []task.Interface{ - haproxyCfg, - getMd5Sum, - haproxyManifestK8s, - updateKubeletConfig, - updateKubeProxyConfig, - updateHostsFile, - } -} - -type KubevipModule struct { - common.KubeModule - Skip bool -} - -func (k *KubevipModule) IsSkip() bool { - return k.Skip -} - -func (k *KubevipModule) Init() { - k.Name = "InternalLoadbalancerModule" - k.Desc = "Install internal load balancer" - - checkVIPAddress := &task.RemoteTask{ - Name: "CheckVIPAddress", - Desc: "Check VIP Address", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(CheckVIPAddress), - Parallel: true, - } - - getInterface := &task.RemoteTask{ - Name: "GetNodeInterface", - Desc: "Get Node Interface", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Action: new(GetInterfaceName), - Parallel: true, - } - - kubevipManifestOnlyFirstMaster := &task.RemoteTask{ - Name: "GenerateKubevipManifest", - Desc: "Generate kubevip manifest at first master", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(GenerateKubevipManifest), - Parallel: true, - } - - kubevipManifestNotFirstMaster := &task.RemoteTask{ - Name: "GenerateKubevipManifest", - Desc: "Generate kubevip manifest at other master", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Prepare: &common.OnlyFirstMaster{Not: true}, - Action: new(GenerateKubevipManifest), - Parallel: true, - } - - if exist, _ := k.BaseModule.PipelineCache.GetMustBool(common.ClusterExist); exist { - k.Tasks = []task.Interface{ - checkVIPAddress, - getInterface, - kubevipManifestNotFirstMaster, - } - } else { - k.Tasks = []task.Interface{ - checkVIPAddress, - getInterface, - kubevipManifestOnlyFirstMaster, - } - } -} - -type K3sHaproxyModule struct { - common.KubeModule - Skip bool -} - -func (k *K3sHaproxyModule) IsSkip() bool { - return k.Skip -} - -func (k *K3sHaproxyModule) Init() { - k.Name = "InternalLoadbalancerModule" - k.Name = "Install internal load balancer" - - haproxyCfg := &task.RemoteTask{ - Name: "GenerateHaproxyConfig", - Desc: "Generate haproxy.cfg", - Hosts: k.Runtime.GetHostsByRole(common.Worker), - Prepare: new(common.OnlyWorker), - Action: &action.Template{ - Template: templates.HaproxyConfig, - Dst: filepath.Join(common.HaproxyDir, templates.HaproxyConfig.Name()), - Data: util.Data{ - "MasterNodes": templates.MasterNodeStr(k.Runtime, k.KubeConf), - "LoadbalancerApiserverPort": k.KubeConf.Cluster.ControlPlaneEndpoint.Port, - "LoadbalancerApiserverHealthcheckPort": 8081, - "KubernetesType": k.KubeConf.Cluster.Kubernetes.Type, - }, - }, - Parallel: true, - } - - // Calculation config md5 as the checksum. - // It will make load balancer reload when config changes. - getMd5Sum := &task.RemoteTask{ - Name: "GetChecksumFromConfig", - Desc: "Calculate the MD5 value according to haproxy.cfg", - Hosts: k.Runtime.GetHostsByRole(common.Worker), - Prepare: new(common.OnlyWorker), - Action: new(GetChecksum), - Parallel: true, - } - - haproxyManifestK3s := &task.RemoteTask{ - Name: "GenerateHaproxyManifestK3s", - Desc: "Generate haproxy manifest", - Hosts: k.Runtime.GetHostsByRole(common.Worker), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyWorker), - new(common.OnlyK3s), - }, - Action: new(GenerateK3sHaproxyManifest), - Parallel: true, - } - - updateK3sConfig := &task.RemoteTask{ - Name: "UpdateK3sConfig", - Desc: "Update k3s config", - Hosts: k.Runtime.GetHostsByRole(common.Worker), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyK3s), - new(updateK3sPrepare), - }, - Action: new(UpdateK3s), - Parallel: true, - Retry: 3, - } - - // UpdateHostsFile is used to update the '/etc/hosts'. Make the 'lb.kubesphere.local' address to set as 127.0.0.1. - // All of the 'admin.conf' and '/.kube/config' will connect to 127.0.0.1:6443. - updateHostsFile := &task.RemoteTask{ - Name: "UpdateHostsFile", - Desc: "Update /etc/hosts", - Hosts: k.Runtime.GetHostsByRole(common.K8s), - Action: new(UpdateHosts), - Parallel: true, - Retry: 3, - } - - k.Tasks = []task.Interface{ - haproxyCfg, - getMd5Sum, - haproxyManifestK3s, - updateK3sConfig, - updateHostsFile, - } -} - -type K3sKubevipModule struct { - common.KubeModule - Skip bool -} - -func (k *K3sKubevipModule) IsSkip() bool { - return k.Skip -} - -func (k *K3sKubevipModule) Init() { - k.Name = "InternalLoadbalancerModule" - k.Name = "Install internal load balancer" - - checkVIPAddress := &task.RemoteTask{ - Name: "CheckVIPAddress", - Desc: "Check VIP Address", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(CheckVIPAddress), - Parallel: true, - } - - createManifestsFolder := &task.RemoteTask{ - Name: "CreateManifestsFolder", - Desc: "Create Manifests Folder", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(CreateManifestsFolder), - Parallel: true, - } - - getInterface := &task.RemoteTask{ - Name: "GetNodeInterface", - Desc: "Get Node Interface", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(GetInterfaceName), - Parallel: true, - } - - kubevipDaemonsetK3s := &task.RemoteTask{ - Name: "GenerateKubevipManifest", - Desc: "Generate kubevip daemoset", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(GenerateK3sKubevipDaemonset), - Parallel: true, - } - - k.Tasks = []task.Interface{ - checkVIPAddress, - createManifestsFolder, - getInterface, - kubevipDaemonsetK3s, - } -} - -type DeleteVIPModule struct { - common.KubeModule - Skip bool -} - -func (k *DeleteVIPModule) IsSkip() bool { - return k.Skip -} - -func (k *DeleteVIPModule) Init() { - k.Name = "DeleteVIPModule" - k.Desc = "Delete VIP" - - getInterface := &task.RemoteTask{ - Name: "GetNodeInterface", - Desc: "Get Node Interface", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Action: new(GetInterfaceName), - Parallel: true, - } - - DeleteVIP := &task.RemoteTask{ - Name: "Delete VIP", - Desc: "Delete the VIP", - Hosts: k.Runtime.GetHostsByRole(common.Master), - Action: new(DeleteVIP), - Parallel: true, - } - - k.Tasks = []task.Interface{ - getInterface, - DeleteVIP, - } -} diff --git a/cmd/kk/pkg/loadbalancer/prepares.go b/cmd/kk/pkg/loadbalancer/prepares.go deleted file mode 100644 index 7dacbf4d..00000000 --- a/cmd/kk/pkg/loadbalancer/prepares.go +++ /dev/null @@ -1,95 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package loadbalancer - -import ( - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -type updateK3sPrepare struct { - common.KubePrepare -} - -func (u *updateK3sPrepare) PreCheck(runtime connector.Runtime) (bool, error) { - exist, err := runtime.GetRunner().FileExist("/etc/systemd/system/k3s.service") - if err != nil { - return false, err - } - - if exist { - if out, err := runtime.GetRunner().SudoCmd("sed -n '/--server=.*/p' /etc/systemd/system/k3s.service", false); err != nil { - return false, err - } else { - if strings.Contains(strings.TrimSpace(out), LocalServer) { - logger.Log.Debugf("do not restart kubelet, /etc/systemd/system/k3s.service content is %s", out) - return false, nil - } - } - } else { - return false, errors.New("Failed to find /etc/systemd/system/k3s.service") - } - return true, nil -} - -type updateKubeletPrepare struct { - common.KubePrepare -} - -func (u *updateKubeletPrepare) PreCheck(runtime connector.Runtime) (bool, error) { - exist, err := runtime.GetRunner().FileExist("/etc/kubernetes/kubelet.conf") - if err != nil { - return false, err - } - - if exist { - if out, err := runtime.GetRunner().SudoCmd("sed -n '/server:.*/p' /etc/kubernetes/kubelet.conf", true); err != nil { - return false, err - } else { - if strings.Contains(strings.TrimSpace(out), LocalServer) { - logger.Log.Debugf("do not restart kubelet, /etc/kubernetes/kubelet.conf content is %s", out) - return false, nil - } - } - } else { - return false, errors.New("Failed to find /etc/kubernetes/kubelet.conf") - } - return true, nil -} - -type updateKubeProxyPrapre struct { - common.KubePrepare -} - -func (u *updateKubeProxyPrapre) PreCheck(runtime connector.Runtime) (bool, error) { - if out, err := runtime.GetRunner().SudoCmd( - "set -o pipefail && /usr/local/bin/kubectl --kubeconfig /etc/kubernetes/admin.conf get configmap kube-proxy -n kube-system -o yaml "+ - "| sed -n '/server:.*/p'", false); err != nil { - return false, err - } else { - if strings.Contains(strings.TrimSpace(out), LocalServer) { - logger.Log.Debugf("do not restart kube-proxy, configmap kube-proxy content is %s", out) - return false, nil - } - } - return true, nil -} diff --git a/cmd/kk/pkg/loadbalancer/tasks.go b/cmd/kk/pkg/loadbalancer/tasks.go deleted file mode 100644 index c1087d57..00000000 --- a/cmd/kk/pkg/loadbalancer/tasks.go +++ /dev/null @@ -1,324 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package loadbalancer - -import ( - "fmt" - "path/filepath" - "strconv" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/loadbalancer/templates" -) - -type GetChecksum struct { - common.KubeAction -} - -func (g *GetChecksum) Execute(runtime connector.Runtime) error { - md5Str, err := runtime.GetRunner().FileMd5(filepath.Join(common.HaproxyDir, "haproxy.cfg")) - if err != nil { - return err - } - host := runtime.RemoteHost() - // type: string - host.GetCache().Set("md5", md5Str) - return nil -} - -type GenerateHaproxyManifest struct { - common.KubeAction -} - -func (g *GenerateHaproxyManifest) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - md5Str, ok := host.GetCache().GetMustString("md5") - if !ok { - return errors.New("get haproxy config md5 sum by host label failed") - } - - templateAction := action.Template{ - Template: templates.HaproxyManifest, - Dst: filepath.Join(common.KubeManifestDir, templates.HaproxyManifest.Name()), - Data: util.Data{ - "HaproxyImage": images.GetImage(runtime, g.KubeConf, "haproxy").ImageName(), - "HealthCheckPort": 8081, - "Checksum": md5Str, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type UpdateK3s struct { - common.KubeAction -} - -func (u *UpdateK3s) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("sed -i 's#--server=.*\"#--server=https://127.0.0.1:%s\"#g' /etc/systemd/system/k3s.service", false); err != nil { - return err - } - if _, err := runtime.GetRunner().SudoCmd("systemctl restart k3s", false); err != nil { - return err - } - return nil -} - -type UpdateKubelet struct { - common.KubeAction -} - -func (u *UpdateKubelet) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "sed -i 's#server:.*#server: https://127.0.0.1:%s#g' /etc/kubernetes/kubelet.conf", - strconv.Itoa(u.KubeConf.Cluster.ControlPlaneEndpoint.Port)), false); err != nil { - return err - } - if _, err := runtime.GetRunner().SudoCmd("systemctl daemon-reload && systemctl restart kubelet", false); err != nil { - return err - } - return nil -} - -type UpdateKubeProxy struct { - common.KubeAction -} - -func (u *UpdateKubeProxy) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "set -o pipefail "+ - "&& /usr/local/bin/kubectl --kubeconfig /etc/kubernetes/admin.conf get configmap kube-proxy -n kube-system -o yaml "+ - "| sed 's#server:.*#server: https://127.0.0.1:%s#g' "+ - "| /usr/local/bin/kubectl --kubeconfig /etc/kubernetes/admin.conf replace -f -", - strconv.Itoa(u.KubeConf.Cluster.ControlPlaneEndpoint.Port)), false); err != nil { - return err - } - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl "+ - "--kubeconfig /etc/kubernetes/admin.conf delete pod "+ - "-n kube-system "+ - "-l k8s-app=kube-proxy "+ - "--force "+ - "--grace-period=0", false); err != nil { - return err - } - return nil -} - -type UpdateHosts struct { - common.KubeAction -} - -func (u *UpdateHosts) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("sed -i 's#.* %s#127.0.0.1 %s#g' /etc/hosts", - u.KubeConf.Cluster.ControlPlaneEndpoint.Domain, u.KubeConf.Cluster.ControlPlaneEndpoint.Domain), false); err != nil { - return err - } - return nil -} - -type CheckVIPAddress struct { - common.KubeAction -} - -func (c *CheckVIPAddress) Execute(runtime connector.Runtime) error { - if c.KubeConf.Cluster.ControlPlaneEndpoint.Address == "" { - return errors.New("VIP address is empty") - } else { - return nil - } -} - -type GetInterfaceName struct { - common.KubeAction -} - -func (g *GetInterfaceName) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - if g.KubeConf.Cluster.ControlPlaneEndpoint.KubeVip.Mode == "BGP" { - host.GetCache().Set("interface", "lo") - return nil - } - cmd := fmt.Sprintf("ip route "+ - "| grep ' %s ' "+ - "| grep 'proto kernel scope link src'"+ - "| sed -e \"s/^.*dev.//\" -e \"s/.proto.*//\""+ - "| uniq ", host.GetAddress()) - interfaceName, err := runtime.GetRunner().SudoCmd(cmd, false) - if err != nil { - return err - } - if interfaceName == "" { - return errors.New("get interface failed") - } - // type: string - host.GetCache().Set("interface", interfaceName) - return nil -} - -type GenerateKubevipManifest struct { - common.KubeAction -} - -func (g *GenerateKubevipManifest) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - interfaceName, ok := host.GetCache().GetMustString("interface") - if !ok { - return errors.New("get interface failed") - } - BGPMode := g.KubeConf.Cluster.ControlPlaneEndpoint.KubeVip.Mode == "BGP" - hosts := runtime.GetHostsByRole(common.Master) - var BGPPeersArr []string - for _, value := range hosts { - address := value.GetAddress() - if address == host.GetAddress() { - continue - } - BGPPeersArr = append(BGPPeersArr, fmt.Sprintf("%s:65000::false", address)) - } - BGPPeers := strings.Join(BGPPeersArr, ",") - templateAction := action.Template{ - Template: templates.KubevipManifest, - Dst: filepath.Join(common.KubeManifestDir, templates.KubevipManifest.Name()), - Data: util.Data{ - "BGPMode": BGPMode, - "VipInterface": interfaceName, - "BGPRouterID": host.GetAddress(), - "BGPPeers": BGPPeers, - "KubeVip": g.KubeConf.Cluster.ControlPlaneEndpoint.Address, - "KubevipImage": images.GetImage(runtime, g.KubeConf, "kubevip").ImageName(), - }, - } - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type GenerateK3sHaproxyManifest struct { - common.KubeAction -} - -func (g *GenerateK3sHaproxyManifest) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - md5Str, ok := host.GetCache().GetMustString("md5") - if !ok { - return errors.New("get haproxy config md5 sum by host label failed") - } - - templateAction := action.Template{ - Template: templates.HaproxyManifest, - Dst: filepath.Join("/var/lib/rancher/k3s/agent/pod-manifests", templates.HaproxyManifest.Name()), - Data: util.Data{ - "HaproxyImage": images.GetImage(runtime, g.KubeConf, "haproxy").ImageName(), - "HealthCheckPort": 8081, - "Checksum": md5Str, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type CreateManifestsFolder struct { - action.BaseAction -} - -func (h *CreateManifestsFolder) Execute(runtime connector.Runtime) error { - _, err := runtime.GetRunner().SudoCmd("mkdir -p /var/lib/rancher/k3s/server/manifests/", false) - if err != nil { - return err - } - return nil -} - -type GenerateK3sKubevipDaemonset struct { - common.KubeAction -} - -func (g *GenerateK3sKubevipDaemonset) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - interfaceName, ok := host.GetCache().GetMustString("interface") - if !ok { - return errors.New("get interface failed") - } - BGPMode := g.KubeConf.Cluster.ControlPlaneEndpoint.KubeVip.Mode == "BGP" - hosts := runtime.GetHostsByRole(common.Master) - var BGPPeersArr []string - for _, value := range hosts { - address := value.GetAddress() - if address == host.GetAddress() { - continue - } - BGPPeersArr = append(BGPPeersArr, fmt.Sprintf("%s:65000::false", address)) - } - BGPPeers := strings.Join(BGPPeersArr, ",") - templateAction := action.Template{ - Template: templates.K3sKubevipManifest, - Dst: filepath.Join("/var/lib/rancher/k3s/server/manifests/", templates.K3sKubevipManifest.Name()), - Data: util.Data{ - "BGPMode": BGPMode, - "KubeVipVersion": images.GetImage(runtime, g.KubeConf, "kubevip").Tag, - "VipInterface": interfaceName, - "BGPRouterID": host.GetAddress(), - "BGPPeers": BGPPeers, - "KubeVip": g.KubeConf.Cluster.ControlPlaneEndpoint.Address, - "KubevipImage": images.GetImage(runtime, g.KubeConf, "kubevip").ImageName(), - }, - } - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type DeleteVIP struct { - common.KubeAction -} - -func (g *DeleteVIP) Execute(runtime connector.Runtime) error { - host := runtime.RemoteHost() - interfaceName, ok := host.GetCache().GetMustString("interface") - if !ok { - return errors.New("get interface failed") - } - - address := host.GetAddress() - internalAddress := host.GetInternalAddress() - if address == g.KubeConf.Cluster.ControlPlaneEndpoint.Address || internalAddress == g.KubeConf.Cluster.ControlPlaneEndpoint.Address { - return nil - } - - cmd := fmt.Sprintf("ip addr del %s dev %s", g.KubeConf.Cluster.ControlPlaneEndpoint.Address, interfaceName) - runtime.GetRunner().SudoCmd(cmd, false) - return nil -} diff --git a/cmd/kk/pkg/loadbalancer/templates/haproxyConfig.go b/cmd/kk/pkg/loadbalancer/templates/haproxyConfig.go deleted file mode 100644 index 921245c6..00000000 --- a/cmd/kk/pkg/loadbalancer/templates/haproxyConfig.go +++ /dev/null @@ -1,83 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "strconv" - "text/template" - - "github.com/lithammer/dedent" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -var HaproxyConfig = template.Must(template.New("haproxy.cfg").Parse( - dedent.Dedent(` -global - maxconn 4000 - log 127.0.0.1 local0 - -defaults - mode http - log global - option httplog - option dontlognull - option http-server-close - option redispatch - retries 5 - timeout http-request 5m - timeout queue 5m - timeout connect 30s - timeout client 30s - timeout server 15m - timeout http-keep-alive 30s - timeout check 30s - maxconn 4000 - -frontend healthz - bind *:{{ .LoadbalancerApiserverHealthcheckPort }} - mode http - monitor-uri /healthz - -frontend kube_api_frontend - bind 127.0.0.1:{{ .LoadbalancerApiserverPort }} - mode tcp - option tcplog - default_backend kube_api_backend - -backend kube_api_backend - mode tcp - balance leastconn - default-server inter 15s downinter 15s rise 2 fall 2 slowstart 60s maxconn 1000 maxqueue 256 weight 100 - {{- if ne .KubernetesType "k3s"}} - option httpchk GET /healthz - {{- end }} - http-check expect status 200 - {{- range .MasterNodes }} - server {{ . }} check check-ssl verify none - {{- end }} -`))) - -func MasterNodeStr(runtime connector.ModuleRuntime, conf *common.KubeConf) []string { - masterNodes := make([]string, len(runtime.GetHostsByRole(common.Master))) - for i, node := range runtime.GetHostsByRole(common.Master) { - masterNodes[i] = node.GetName() + " " + node.GetAddress() + ":" + strconv.Itoa(kubekeyapiv1alpha2.DefaultApiserverPort) - } - return masterNodes -} diff --git a/cmd/kk/pkg/loadbalancer/templates/haproxyManifest.go b/cmd/kk/pkg/loadbalancer/templates/haproxyManifest.go deleted file mode 100644 index d4f5f715..00000000 --- a/cmd/kk/pkg/loadbalancer/templates/haproxyManifest.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var HaproxyManifest = template.Must(template.New("haproxy.yaml").Parse( - dedent.Dedent(` -apiVersion: v1 -kind: Pod -metadata: - name: haproxy - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: Reconcile - k8s-app: kube-haproxy - annotations: - cfg-checksum: "{{ .Checksum }}" -spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - nodeSelector: - kubernetes.io/os: linux - priorityClassName: system-node-critical - containers: - - name: haproxy - image: {{ .HaproxyImage }} - imagePullPolicy: IfNotPresent - resources: - requests: - cpu: 25m - memory: 32M - livenessProbe: - httpGet: - path: /healthz - port: {{ .HealthCheckPort }} - readinessProbe: - httpGet: - path: /healthz - port: {{ .HealthCheckPort }} - volumeMounts: - - mountPath: /usr/local/etc/haproxy/ - name: etc-haproxy - readOnly: true - volumes: - - name: etc-haproxy - hostPath: - path: /etc/kubekey/haproxy -`))) diff --git a/cmd/kk/pkg/loadbalancer/templates/k3sKubevipManifest.go b/cmd/kk/pkg/loadbalancer/templates/k3sKubevipManifest.go deleted file mode 100644 index 827d4af4..00000000 --- a/cmd/kk/pkg/loadbalancer/templates/k3sKubevipManifest.go +++ /dev/null @@ -1,279 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var K3sKubevipManifest = template.Must(template.New("kube-vip-rbac.yaml").Parse( - dedent.Dedent(`{{ if .BGPMode }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-vip - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role -rules: - - apiGroups: [""] - resources: ["services", "services/status", "nodes", "endpoints"] - verbs: ["list","get","watch", "update"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: system:kube-vip-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-vip-role -subjects: -- kind: ServiceAccount - name: kube-vip - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: {{ .KubeVipVersion }} - name: kube-vip-ds - namespace: kube-system -spec: - selector: - matchLabels: - app.kubernetes.io/name: kube-vip-ds - template: - metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: {{ .KubeVipVersion }} - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - containers: - - args: - - manager - env: - - name: vip_arp - value: "false" - - name: port - value: "6443" - - name: vip_interface - value: {{ .VipInterface }} - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: bgp_enable - value: "true" - - name: bgp_routerid - value: {{ .BGPRouterID }} - - name: bgp_as - value: "65000" - - name: bgp_peeraddress - - name: bgp_peerpass - - name: bgp_peeras - value: "65000" - - name: bgp_peers - value: {{ .BGPPeers }} - - name: lb_enable - value: "true" - - name: lb_port - value: "6443" - - name: lb_fwdmethod - value: local - - name: address - value: {{ .KubeVip }} - - name: prometheus_server - value: :2112 - image: {{ .KubevipImage }} - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - hostNetwork: true - serviceAccountName: kube-vip - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - updateStrategy: {} -status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 -{{ else }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-vip - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role -rules: - - apiGroups: [""] - resources: ["services", "services/status", "nodes", "endpoints"] - verbs: ["list","get","watch", "update"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: system:kube-vip-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-vip-role -subjects: -- kind: ServiceAccount - name: kube-vip - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: {{ .KubeVipVersion }} - name: kube-vip-ds - namespace: kube-system -spec: - selector: - matchLabels: - app.kubernetes.io/name: kube-vip-ds - template: - metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: {{ .KubeVipVersion }} - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_interface - value: {{ .VipInterface }} - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: lb_enable - value: "true" - - name: lb_port - value: "6443" - - name: lb_fwdmethod - value: local - - name: address - value: {{ .KubeVip }} - - name: prometheus_server - value: :2112 - image: {{ .KubevipImage }} - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - hostNetwork: true - serviceAccountName: kube-vip - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - updateStrategy: {} -status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 -{{ end }} -`))) diff --git a/cmd/kk/pkg/loadbalancer/templates/kubevipManifest.go b/cmd/kk/pkg/loadbalancer/templates/kubevipManifest.go deleted file mode 100644 index 111e418c..00000000 --- a/cmd/kk/pkg/loadbalancer/templates/kubevipManifest.go +++ /dev/null @@ -1,164 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var KubevipManifest = template.Must(template.New("kube-vip.yaml").Parse( - dedent.Dedent(`{{ if .BGPMode }} -apiVersion: v1 -kind: Pod -metadata: - creationTimestamp: null - name: kube-vip - namespace: kube-system -spec: - containers: - - args: - - manager - env: - - name: vip_arp - value: "false" - - name: port - value: "6443" - - name: vip_interface - value: {{ .VipInterface }} - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: bgp_enable - value: "true" - - name: bgp_routerid - value: {{ .BGPRouterID }} - - name: bgp_as - value: "65000" - - name: bgp_peeraddress - - name: bgp_peerpass - - name: bgp_peeras - value: "65000" - - name: bgp_peers - value: {{ .BGPPeers }} - - name: lb_enable - value: "true" - - name: lb_port - value: "6443" - - name: lb_fwdmethod - value: local - - name: address - value: {{ .KubeVip }} - - name: prometheus_server - value: :2112 - image: {{ .KubevipImage }} - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - hostNetwork: true - volumes: - - hostPath: - path: /etc/kubernetes/admin.conf - name: kubeconfig -status: {} -{{ else }} -apiVersion: v1 -kind: Pod -metadata: - creationTimestamp: null - name: kube-vip - namespace: kube-system -spec: - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_interface - value: {{ .VipInterface }} - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: lb_enable - value: "true" - - name: lb_port - value: "6443" - - name: address - value: {{ .KubeVip }} - image: {{ .KubevipImage }} - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - - SYS_TIME - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - hostNetwork: true - volumes: - - hostPath: - path: /etc/kubernetes/admin.conf - name: kubeconfig -status: {} -{{ end }} -`))) diff --git a/cmd/kk/pkg/phase/artifact/artifact.go b/cmd/kk/pkg/phase/artifact/artifact.go deleted file mode 100755 index 5d540ca8..00000000 --- a/cmd/kk/pkg/phase/artifact/artifact.go +++ /dev/null @@ -1,64 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package artifact - -import ( - "errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" -) - -func NewArtifactImportPipeline(runtime *common.KubeRuntime) error { - - m := []module.Module{ - &artifact.UnArchiveModule{}, - } - - p := pipeline.Pipeline{ - Name: "ArtifactImportPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func ArtifactImport(args common.Argument) error { - var loaderType string - - loaderType = common.AllInOne - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewArtifactImportPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/binary/create_binary.go b/cmd/kk/pkg/phase/binary/create_binary.go deleted file mode 100755 index b29987e1..00000000 --- a/cmd/kk/pkg/phase/binary/create_binary.go +++ /dev/null @@ -1,78 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binary - -import ( - "errors" - "fmt" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" -) - -func NewCreateBinaryPipeline(runtime *common.KubeRuntime) error { - - m := []module.Module{ - &precheck.NodePreCheckModule{}, - &binaries.NodeBinariesModule{}, - &SyncBinaryModule{}, - } - - p := pipeline.Pipeline{ - Name: "CreateBinaryPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func CreateBinary(args common.Argument, downloadCmd string) error { - args.DownloadCommand = func(path, url string) string { - // this is an extension point for downloading tools, for example users can set the timeout, proxy or retry under - // some poor network environment. Or users even can choose another cli, it might be wget. - // perhaps we should have a build-in download function instead of totally rely on the external one - return fmt.Sprintf(downloadCmd, path, url) - } - var loaderType string - - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewCreateBinaryPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/binary/modules.go b/cmd/kk/pkg/phase/binary/modules.go deleted file mode 100755 index 40208c8f..00000000 --- a/cmd/kk/pkg/phase/binary/modules.go +++ /dev/null @@ -1,45 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binary - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" -) - -type SyncBinaryModule struct { - common.KubeModule -} - -func (p *SyncBinaryModule) Init() { - p.Name = "SyncBinaryModule" - p.Desc = "synchronize kubernetes binaries" - - syncBinary := &task.RemoteTask{ - Name: "SyncKubeBinary", - Desc: "Synchronize kubernetes binaries", - Hosts: p.Runtime.GetHostsByRole(common.K8s), - Action: new(kubernetes.SyncKubeBinary), - Parallel: true, - Retry: 2, - } - - p.Tasks = []task.Interface{ - syncBinary, - } -} diff --git a/cmd/kk/pkg/phase/binary/task.go b/cmd/kk/pkg/phase/binary/task.go deleted file mode 100755 index 4300c659..00000000 --- a/cmd/kk/pkg/phase/binary/task.go +++ /dev/null @@ -1,67 +0,0 @@ -package binary - -import ( - "fmt" - - "github.com/pkg/errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" -) - -type GetBinaryPath struct { - common.KubeAction - Binaries []string -} - -func (g *GetBinaryPath) Execute(runtime connector.Runtime) error { - cfg := g.KubeConf.Cluster - - archMap := make(map[string]bool) - for _, host := range cfg.Hosts { - switch host.Arch { - case "amd64": - archMap["amd64"] = true - case "arm64": - archMap["arm64"] = true - default: - return errors.New(fmt.Sprintf("Unsupported architecture: %s", host.Arch)) - } - } - - for arch := range archMap { - if err := setBinaryPath(g.KubeConf, runtime.GetWorkDir(), arch, g.Binaries, g.PipelineCache); err != nil { - return err - } - } - return nil -} - -func setBinaryPath(kubeConf *common.KubeConf, path, arch string, binaries []string, pipelineCache *cache.Cache) error { - binariesMap := make(map[string]*files.KubeBinary) - var kubeBinary *files.KubeBinary - for _, binary := range binaries { - switch binary { - case "etcd": - kubeBinary = files.NewKubeBinary(binary, arch, kubekeyapiv1alpha2.DefaultEtcdVersion, path, kubeConf.Arg.DownloadCommand) - case "docker": - kubeBinary = files.NewKubeBinary(binary, arch, kubekeyapiv1alpha2.DefaultDockerVersion, path, kubeConf.Arg.DownloadCommand) - case "containerd": - kubeBinary = files.NewKubeBinary(binary, arch, kubekeyapiv1alpha2.DefaultContainerdVersion, path, kubeConf.Arg.DownloadCommand) - case "helm": - kubeBinary = files.NewKubeBinary(binary, arch, kubekeyapiv1alpha2.DefaultHelmVersion, path, kubeConf.Arg.DownloadCommand) - case "crictl": - kubeBinary = files.NewKubeBinary("crictl", arch, kubekeyapiv1alpha2.DefaultCrictlVersion, path, kubeConf.Arg.DownloadCommand) - case "runc": - kubeBinary = files.NewKubeBinary("runc", arch, kubekeyapiv1alpha2.DefaultRuncVersion, path, kubeConf.Arg.DownloadCommand) - default: - return errors.New(fmt.Sprintf("Unsupported binary name: %s", binary)) - } - binariesMap[kubeBinary.ID] = kubeBinary - } - pipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap) - return nil -} diff --git a/cmd/kk/pkg/phase/binary/upgrade_binary.go b/cmd/kk/pkg/phase/binary/upgrade_binary.go deleted file mode 100755 index e1639ed6..00000000 --- a/cmd/kk/pkg/phase/binary/upgrade_binary.go +++ /dev/null @@ -1,78 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package binary - -import ( - "errors" - "fmt" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/precheck" -) - -func NewUpgradeBinaryPipeline(runtime *common.KubeRuntime) error { - - m := []module.Module{ - &precheck.UpgradePreCheckModule{}, - &binaries.NodeBinariesModule{}, - &SyncBinaryModule{}, - } - - p := pipeline.Pipeline{ - Name: "UpgradeBinaryPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func UpgradeBinary(args common.Argument, downloadCmd string) error { - args.DownloadCommand = func(path, url string) string { - // this is an extension point for downloading tools, for example users can set the timeout, proxy or retry under - // some poor network environment. Or users even can choose another cli, it might be wget. - // perhaps we should have a build-in download function instead of totally rely on the external one - return fmt.Sprintf(downloadCmd, path, url) - } - var loaderType string - - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewUpgradeBinaryPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/confirm/confirm.go b/cmd/kk/pkg/phase/confirm/confirm.go deleted file mode 100644 index fe847d3e..00000000 --- a/cmd/kk/pkg/phase/confirm/confirm.go +++ /dev/null @@ -1,82 +0,0 @@ -package confirm - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type UpgradeK8sConfirmModule struct { - common.KubeModule -} - -func (u *UpgradeK8sConfirmModule) Init() { - u.Name = "UpgradeKsConfirmModule" - u.Desc = "Display upgrade kubesphere confirmation form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: new(UpgradeK8sConfirm), - } - - u.Tasks = []task.Interface{ - display, - } -} - -type UpgradeKsConfirmModule struct { - common.KubeModule -} - -func (u *UpgradeKsConfirmModule) Init() { - u.Name = "UpgradeKsConfirmModule" - u.Desc = "Display upgrade kubesphere confirmation form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: new(UpgradeKsConfirm), - } - - u.Tasks = []task.Interface{ - display, - } -} - -type CreateK8sConfirmModule struct { - common.KubeModule -} - -func (u *CreateK8sConfirmModule) Init() { - u.Name = "CreateKsConfirmModule" - u.Desc = "Display Create kubesphere confirmation form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: new(CreateK8sConfirm), - } - - u.Tasks = []task.Interface{ - display, - } -} - -type CreateKsConfirmModule struct { - common.KubeModule -} - -func (u *CreateKsConfirmModule) Init() { - u.Name = "CreateKsConfirmModule" - u.Desc = "Display Create kubesphere confirmation form" - - display := &task.LocalTask{ - Name: "ConfirmForm", - Desc: "Display confirmation form", - Action: new(CreateKsConfirm), - } - - u.Tasks = []task.Interface{ - display, - } -} diff --git a/cmd/kk/pkg/phase/confirm/task.go b/cmd/kk/pkg/phase/confirm/task.go deleted file mode 100644 index 365e31ae..00000000 --- a/cmd/kk/pkg/phase/confirm/task.go +++ /dev/null @@ -1,302 +0,0 @@ -package confirm - -import ( - "bufio" - "errors" - "fmt" - "os" - "strings" - - "github.com/mitchellh/mapstructure" - "github.com/modood/table" - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -type UpgradeK8sConfirm struct { - common.KubeAction -} - -func (u *UpgradeK8sConfirm) Execute(runtime connector.Runtime) error { - pre := make([]map[string]string, len(runtime.GetAllHosts()), len(runtime.GetAllHosts())) - for i, host := range runtime.GetAllHosts() { - if v, ok := host.GetCache().Get(common.NodePreCheck); ok { - pre[i] = v.(map[string]string) - } else { - return errors.New("get node check result failed by host cache") - } - } - - results := make([]confirm.PreCheckResults, len(pre), len(pre)) - for i := range pre { - var result confirm.PreCheckResults - _ = mapstructure.Decode(pre[i], &result) - results[i] = result - } - table.OutputA(results) - fmt.Println() - - warningFlag := false - cmp, err := versionutil.MustParseSemantic(u.KubeConf.Cluster.Kubernetes.Version).Compare("v1.19.0") - if err != nil { - logger.Log.Fatalf("Failed to compare kubernetes version: %v", err) - } - if cmp == 0 || cmp == 1 { - for _, result := range results { - if len(result.Docker) != 0 { - dockerVersion, err := confirm.RefineDockerVersion(result.Docker) - if err != nil { - logger.Log.Fatalf("Failed to get docker version: %v", err) - } - cmp, err := versionutil.MustParseSemantic(dockerVersion).Compare("20.10.0") - if err != nil { - logger.Log.Fatalf("Failed to compare docker version: %v", err) - } - warningFlag = warningFlag || (cmp == -1) - } - } - if warningFlag { - fmt.Println(` -Warning: - - An old Docker version may cause the failure of upgrade. It is recommended that you upgrade Docker to 20.10+ beforehand. - - Issue: https://github.com/kubernetes/kubernetes/issues/101056`) - fmt.Print("\n") - } - } - - nodeStats, ok := u.PipelineCache.GetMustString(common.ClusterNodeStatus) - if !ok { - return errors.New("get cluster nodes status failed by pipeline cache") - } - fmt.Println("Cluster nodes status:") - fmt.Println(nodeStats + "\n") - - fmt.Println("Upgrade Confirmation:") - currentK8sVersion, ok := u.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return errors.New("get current Kubernetes version failed by pipeline cache") - } - fmt.Printf("kubernetes version: %s to %s\n", currentK8sVersion, u.KubeConf.Cluster.Kubernetes.Version) - - fmt.Println() - - if k8sVersion, err := versionutil.ParseGeneric(u.KubeConf.Cluster.Kubernetes.Version); err == nil { - if cri, ok := u.PipelineCache.GetMustString(common.ClusterNodeCRIRuntimes); ok { - k8sV124 := versionutil.MustParseSemantic("v1.24.0") - if k8sVersion.AtLeast(k8sV124) && versionutil.MustParseSemantic(currentK8sVersion).LessThan(k8sV124) && strings.Contains(cri, "docker") { - fmt.Println("[Notice]") - fmt.Println("Pre-upgrade check failed. The container runtime of the current cluster is Docker.") - fmt.Println("Kubernetes v1.24 and later no longer support dockershim and Docker.") - fmt.Println("Make sure you have completed the migration from Docker to other container runtimes that are compatible with the Kubernetes CRI.") - fmt.Println("For more information, see:") - fmt.Println("https://kubernetes.io/docs/setup/production-environment/container-runtimes/#container-runtimes") - fmt.Println("https://kubernetes.io/blog/2022/02/17/dockershim-faq/") - fmt.Println("") - } - } - } - - reader := bufio.NewReader(os.Stdin) - confirmOK := false - for !confirmOK { - fmt.Printf("Continue upgrading kubernetes? [yes/no]: ") - input, err := reader.ReadString('\n') - if err != nil { - return err - } - input = strings.ToLower(strings.TrimSpace(input)) - - switch input { - case "yes", "y": - confirmOK = true - case "no", "n": - os.Exit(0) - default: - continue - } - } - return nil -} - -type UpgradeKsConfirm struct { - common.KubeAction -} - -func (u *UpgradeKsConfirm) Execute(runtime connector.Runtime) error { - pre := make([]map[string]string, len(runtime.GetAllHosts()), len(runtime.GetAllHosts())) - for i, host := range runtime.GetAllHosts() { - if v, ok := host.GetCache().Get(common.NodePreCheck); ok { - pre[i] = v.(map[string]string) - } else { - return errors.New("get node check result failed by host cache") - } - } - - if u.KubeConf.Cluster.KubeSphere.Enabled { - currentKsVersion, ok := u.PipelineCache.GetMustString(common.KubeSphereVersion) - if !ok { - return errors.New("get current KubeSphere version failed by pipeline cache") - } - fmt.Printf("kubesphere version: %s to %s\n", currentKsVersion, u.KubeConf.Cluster.KubeSphere.Version) - } - fmt.Println() - - reader := bufio.NewReader(os.Stdin) - confirmOK := false - for !confirmOK { - fmt.Printf("Continue upgrading KubeSphere? [yes/no]: ") - input, err := reader.ReadString('\n') - if err != nil { - return err - } - input = strings.ToLower(strings.TrimSpace(input)) - - switch input { - case "yes", "y": - confirmOK = true - case "no", "n": - os.Exit(0) - default: - continue - } - } - return nil -} - -type CreateK8sConfirm struct { - common.KubeAction -} - -func (c *CreateK8sConfirm) Execute(runtime connector.Runtime) error { - var ( - results []confirm.PreCheckResults - stopFlag bool - ) - - pre := make([]map[string]string, 0, len(runtime.GetAllHosts())) - for _, host := range runtime.GetAllHosts() { - if v, ok := host.GetCache().Get(common.NodePreCheck); ok { - pre = append(pre, v.(map[string]string)) - } else { - return errors.New("get node check result failed by host cache") - } - } - - for node := range pre { - var result confirm.PreCheckResults - _ = mapstructure.Decode(pre[node], &result) - results = append(results, result) - } - table.OutputA(results) - reader := bufio.NewReader(os.Stdin) - - if c.KubeConf.Arg.Artifact == "" { - for _, host := range results { - if host.Sudo == "" { - logger.Log.Errorf("%s: sudo is required.", host.Name) - stopFlag = true - } - - if host.Conntrack == "" { - logger.Log.Errorf("%s: conntrack is required.", host.Name) - stopFlag = true - } - - if host.Socat == "" { - logger.Log.Errorf("%s: socat is required.", host.Name) - stopFlag = true - } - } - } - - fmt.Println("") - fmt.Println("This is a simple check of your environment.") - fmt.Println("Before installation, ensure that your machines meet all requirements specified at") - fmt.Println("https://github.com/kubesphere/kubekey#requirements-and-recommendations") - fmt.Println("") - - if k8sVersion, err := versionutil.ParseGeneric(c.KubeConf.Cluster.Kubernetes.Version); err == nil { - if k8sVersion.AtLeast(versionutil.MustParseSemantic("v1.24.0")) && c.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker { - fmt.Println("[Notice]") - fmt.Println("Incorrect runtime. Please specify a container runtime other than Docker to install Kubernetes v1.24 or later.") - fmt.Println("You can set \"spec.kubernetes.containerManager\" in the configuration file to \"containerd\" or add \"--container-manager containerd\" to the \"./kk create cluster\" command.") - fmt.Println("For more information, see:") - fmt.Println("https://github.com/kubesphere/kubekey/blob/master/docs/commands/kk-create-cluster.md") - fmt.Println("https://kubernetes.io/docs/setup/production-environment/container-runtimes/#container-runtimes") - fmt.Println("https://kubernetes.io/blog/2022/02/17/dockershim-faq/") - fmt.Println("") - stopFlag = true - } - } - - if stopFlag { - os.Exit(1) - } - - confirmOK := false - for !confirmOK { - fmt.Printf("Continue this init the cluster? [yes/no]: ") - input, err := reader.ReadString('\n') - if err != nil { - logger.Log.Fatal(err) - } - input = strings.TrimSpace(strings.ToLower(input)) - - switch strings.ToLower(input) { - case "yes", "y": - confirmOK = true - case "no", "n": - os.Exit(0) - default: - continue - } - } - return nil -} - -type CreateKsConfirm struct { - common.KubeAction -} - -func (u *CreateKsConfirm) Execute(runtime connector.Runtime) error { - pre := make([]map[string]string, len(runtime.GetAllHosts()), len(runtime.GetAllHosts())) - for i, host := range runtime.GetAllHosts() { - if v, ok := host.GetCache().Get(common.NodePreCheck); ok { - pre[i] = v.(map[string]string) - } else { - return errors.New("get node check result failed by host cache") - } - } - - if u.KubeConf.Cluster.KubeSphere.Enabled { - fmt.Printf("desired kubesphere version: %s\n", u.KubeConf.Cluster.KubeSphere.Version) - } - fmt.Println() - - reader := bufio.NewReader(os.Stdin) - confirmOK := false - for !confirmOK { - fmt.Printf("Continue install KubeSphere? [yes/no]: ") - input, err := reader.ReadString('\n') - if err != nil { - return err - } - input = strings.ToLower(strings.TrimSpace(input)) - - switch input { - case "yes", "y": - confirmOK = true - case "no", "n": - os.Exit(0) - default: - continue - } - } - return nil -} diff --git a/cmd/kk/pkg/phase/etcd/etcd.go b/cmd/kk/pkg/phase/etcd/etcd.go deleted file mode 100755 index ce40781f..00000000 --- a/cmd/kk/pkg/phase/etcd/etcd.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package etcd - -import ( - "errors" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/etcd" -) - -func NewCreateEtcdPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - } - - p := pipeline.Pipeline{ - Name: "CreateEtcdPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func CreateEtcd(args common.Argument) error { - var loaderType string - - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewCreateEtcdPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/etcd/modules.go b/cmd/kk/pkg/phase/etcd/modules.go deleted file mode 100755 index 7e5ca11f..00000000 --- a/cmd/kk/pkg/phase/etcd/modules.go +++ /dev/null @@ -1,42 +0,0 @@ -package etcd - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/etcd" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/binary" -) - -type PreCheckModule struct { - common.KubeModule - Skip bool -} - -func (p *PreCheckModule) IsSkip() bool { - return p.Skip -} - -func (p *PreCheckModule) Init() { - p.Name = "ETCDPreCheckModule" - p.Desc = "Get ETCD cluster status" - - getStatus := &task.RemoteTask{ - Name: "GetETCDStatus", - Desc: "Get etcd status", - Hosts: p.Runtime.GetHostsByRole(common.ETCD), - Action: new(etcd.GetStatus), - Parallel: false, - Retry: 0, - } - - setBinaryCache := &task.LocalTask{ - Name: "SetEtcdBinaryCache", - Desc: "Set Etcd Binary Path in PipelineCache", - Action: &binary.GetBinaryPath{Binaries: []string{"etcd"}}, - } - - p.Tasks = []task.Interface{ - getStatus, - setBinaryCache, - } -} diff --git a/cmd/kk/pkg/phase/images/create_images.go b/cmd/kk/pkg/phase/images/create_images.go deleted file mode 100755 index 6bbb0f91..00000000 --- a/cmd/kk/pkg/phase/images/create_images.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" -) - -func NewCreateImagesPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.NodePreCheckModule{}, - &setBinaryCacheModule{}, - &kubernetes.StatusModule{}, - &container.InstallContainerModule{}, - } - - p := pipeline.Pipeline{ - Name: "CreateImagesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func CreateImages(args common.Argument) error { - var loaderType string - - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewCreateImagesPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/images/modules.go b/cmd/kk/pkg/phase/images/modules.go deleted file mode 100644 index 4fe93748..00000000 --- a/cmd/kk/pkg/phase/images/modules.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/binary" -) - -type UpgradeImagesModule struct { - common.KubeModule -} - -func (p *UpgradeImagesModule) Init() { - p.Name = "UpgradeImagesModule" - p.Desc = "pull the images that cluster need" - - pull := &task.RemoteTask{ - Name: "PullImages", - Desc: "Start to pull images on all nodes", - Hosts: p.Runtime.GetHostsByRole(common.K8s), - Prepare: new(kubernetes.NotEqualPlanVersion), - Action: new(images.PullImage), - Parallel: true, - } - - p.Tasks = []task.Interface{ - pull, - } -} - -type setBinaryCacheModule struct { - common.KubeModule -} - -func (p *setBinaryCacheModule) Init() { - p.Name = "setBinaryCacheModule" - p.Desc = "set the docker and containerd binary paths in cache" - - setBinaryCache := &task.LocalTask{ - Name: "SetBinaryCache", - Desc: "Set Binary Path in PipelineCache", - Action: &binary.GetBinaryPath{Binaries: []string{"docker", "containerd", "runc", "crictl"}}, - } - - p.Tasks = []task.Interface{ - setBinaryCache, - } -} diff --git a/cmd/kk/pkg/phase/images/upgrade_images.go b/cmd/kk/pkg/phase/images/upgrade_images.go deleted file mode 100755 index 65016684..00000000 --- a/cmd/kk/pkg/phase/images/upgrade_images.go +++ /dev/null @@ -1,69 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package images - -import ( - "errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/precheck" -) - -func NewUpgradeImagesPipeline(runtime *common.KubeRuntime) error { - - m := []module.Module{ - &precheck.UpgradePreCheckModule{}, - &UpgradeImagesModule{}, - } - - p := pipeline.Pipeline{ - Name: "UpgradeImagesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func UpgradeImages(args common.Argument) error { - var loaderType string - - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewUpgradeImagesPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/kubernetes/configure.go b/cmd/kk/pkg/phase/kubernetes/configure.go deleted file mode 100755 index 82e82bf9..00000000 --- a/cmd/kk/pkg/phase/kubernetes/configure.go +++ /dev/null @@ -1,89 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/addons" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/network" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/storage" -) - -func NewCreateConfigureKubernetesPipeline(runtime *common.KubeRuntime) error { - skipLocalStorage := true - if runtime.Arg.DeployLocalStorage != nil { - skipLocalStorage = !*runtime.Arg.DeployLocalStorage - } else if runtime.Cluster.KubeSphere.Enabled { - skipLocalStorage = false - } - m := []module.Module{ - &precheck.NodePreCheckModule{}, - &kubernetes.StatusModule{}, - &network.DeployNetworkPluginModule{}, - &kubernetes.ConfigureKubernetesModule{}, - &filesystem.ChownModule{}, - &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, - &kubernetes.SaveKubeConfigModule{}, - &plugins.DeployPluginsModule{}, - &addons.AddonsModule{}, - &storage.DeployLocalVolumeModule{Skip: skipLocalStorage}, - } - - p := pipeline.Pipeline{ - Name: "CreateConfigureKubernetesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func CreateConfigureKubernetes(args common.Argument) error { - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewCreateConfigureKubernetesPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/kubernetes/init.go b/cmd/kk/pkg/phase/kubernetes/init.go deleted file mode 100644 index 9d11bf23..00000000 --- a/cmd/kk/pkg/phase/kubernetes/init.go +++ /dev/null @@ -1,76 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns" -) - -func NewCreateInitClusterPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.NodePreCheckModule{}, - &kubernetes.StatusModule{}, - &confirm.CreateK8sConfirmModule{}, - &InstallKubeletModule{}, - &kubernetes.InitKubernetesModule{}, - &dns.ClusterDNSModule{}, - } - - p := pipeline.Pipeline{ - Name: "CreateInitClusterPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func CreateInitCluster(args common.Argument) error { - - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewCreateInitClusterPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/kubernetes/join.go b/cmd/kk/pkg/phase/kubernetes/join.go deleted file mode 100644 index 75c1bfb6..00000000 --- a/cmd/kk/pkg/phase/kubernetes/join.go +++ /dev/null @@ -1,70 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" -) - -func NewCreateJoinNodesPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.NodePreCheckModule{}, - &kubernetes.StatusModule{}, - &kubernetes.JoinNodesModule{}, - } - - p := pipeline.Pipeline{ - Name: "CreateJoinNodesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func CreateJoinNodes(args common.Argument) error { - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewCreateJoinNodesPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/kubernetes/modules.go b/cmd/kk/pkg/phase/kubernetes/modules.go deleted file mode 100644 index 4700d2f3..00000000 --- a/cmd/kk/pkg/phase/kubernetes/modules.go +++ /dev/null @@ -1,70 +0,0 @@ -package kubernetes - -import ( - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes/templates" -) - -type InstallKubeletModule struct { - common.KubeModule -} - -func (i *InstallKubeletModule) Init() { - i.Name = "InstallKubeletModule" - i.Desc = "Install kubernetes cluster" - - chmodKubelet := &task.RemoteTask{ - Name: "ChmodKubelet", - Desc: "Change kubelet mode", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &kubernetes.NodeInCluster{Not: true}, - Action: new(kubernetes.ChmodKubelet), - Parallel: true, - Retry: 2, - } - - generateKubeletService := &task.RemoteTask{ - Name: "GenerateKubeletService", - Desc: "Generate kubelet service", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &kubernetes.NodeInCluster{Not: true}, - Action: &action.Template{ - Template: templates.KubeletService, - Dst: filepath.Join("/etc/systemd/system/", templates.KubeletService.Name()), - }, - Parallel: true, - Retry: 2, - } - - enableKubelet := &task.RemoteTask{ - Name: "EnableKubelet", - Desc: "Enable kubelet service", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &kubernetes.NodeInCluster{Not: true}, - Action: new(kubernetes.EnableKubelet), - Parallel: true, - Retry: 5, - } - - generateKubeletEnv := &task.RemoteTask{ - Name: "GenerateKubeletEnv", - Desc: "Generate kubelet env", - Hosts: i.Runtime.GetHostsByRole(common.K8s), - Prepare: &kubernetes.NodeInCluster{Not: true}, - Action: new(kubernetes.GenerateKubeletEnv), - Parallel: true, - Retry: 2, - } - - i.Tasks = []task.Interface{ - chmodKubelet, - generateKubeletService, - enableKubelet, - generateKubeletEnv, - } -} diff --git a/cmd/kk/pkg/phase/kubesphere/create_kubesphere.go b/cmd/kk/pkg/phase/kubesphere/create_kubesphere.go deleted file mode 100755 index 76dd6286..00000000 --- a/cmd/kk/pkg/phase/kubesphere/create_kubesphere.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package alpha - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubesphere" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/confirm" -) - -func NewCreateKubeSpherePipeline(runtime *common.KubeRuntime) error { - - m := []module.Module{ - &precheck.NodePreCheckModule{}, - &confirm.CreateKsConfirmModule{}, - &kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - } - - p := pipeline.Pipeline{ - Name: "CreateKubeSpherePipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func CreateKubeSphere(args common.Argument) error { - - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewCreateKubeSpherePipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/kubesphere/upgrade_kubesphere.go b/cmd/kk/pkg/phase/kubesphere/upgrade_kubesphere.go deleted file mode 100755 index 8eebf190..00000000 --- a/cmd/kk/pkg/phase/kubesphere/upgrade_kubesphere.go +++ /dev/null @@ -1,77 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package alpha - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubesphere" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/precheck" -) - -func NewUpgradeKubeSpherePipeline(runtime *common.KubeRuntime) error { - - m := []module.Module{ - &precheck.UpgradeKubeSpherePreCheckModule{}, - &precheck.UpgradeksPhaseDependencyCheckModule{}, - &confirm.UpgradeKsConfirmModule{}, - &kubesphere.CleanClusterConfigurationModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.ConvertModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - } - - p := pipeline.Pipeline{ - Name: "UpgradeKubeSpherePipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func UpgradeKubeSphere(args common.Argument) error { - - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewUpgradeKubeSpherePipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/nodes/modules.go b/cmd/kk/pkg/phase/nodes/modules.go deleted file mode 100644 index 90532926..00000000 --- a/cmd/kk/pkg/phase/nodes/modules.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package nodes - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" -) - -type UpgradeNodesModule struct { - common.KubeModule -} - -func (p *UpgradeNodesModule) Init() { - p.Name = "UpgradeNodesModule" - p.Desc = "Upgrade cluster on all nodes" - - upgradeKubeMaster := &task.RemoteTask{ - Name: "UpgradeClusterOnMaster", - Desc: "Upgrade cluster on master", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: new(kubernetes.NotEqualPlanVersion), - Action: &kubernetes.UpgradeKubeMaster{ModuleName: p.Name}, - Parallel: false, - } - - cluster := kubernetes.NewKubernetesStatus() - p.PipelineCache.GetOrSet(common.ClusterStatus, cluster) - - clusterStatus := &task.RemoteTask{ - Name: "GetClusterStatus", - Desc: "Get kubernetes cluster status", - Hosts: p.Runtime.GetHostsByRole(common.Master), - Prepare: new(kubernetes.NotEqualPlanVersion), - Action: new(kubernetes.GetClusterStatus), - Parallel: false, - } - - upgradeNodes := &task.RemoteTask{ - Name: "UpgradeClusterOnWorker", - Desc: "Upgrade cluster on worker", - Hosts: p.Runtime.GetHostsByRole(common.Worker), - Prepare: &prepare.PrepareCollection{ - new(kubernetes.NotEqualPlanVersion), - new(common.OnlyWorker), - }, - Action: &kubernetes.UpgradeKubeWorker{ModuleName: p.Name}, - Parallel: false, - } - - p.Tasks = []task.Interface{ - upgradeKubeMaster, - clusterStatus, - upgradeNodes, - } -} diff --git a/cmd/kk/pkg/phase/nodes/nodes.go b/cmd/kk/pkg/phase/nodes/nodes.go deleted file mode 100755 index ddd7f27f..00000000 --- a/cmd/kk/pkg/phase/nodes/nodes.go +++ /dev/null @@ -1,71 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package nodes - -import ( - "errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/phase/precheck" -) - -func NewUpgradeNodesPipeline(runtime *common.KubeRuntime) error { - - m := []module.Module{ - &precheck.UpgradePreCheckModule{}, - &confirm.UpgradeK8sConfirmModule{}, - &UpgradeNodesModule{}, - } - - p := pipeline.Pipeline{ - Name: "UpgradeNodesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func UpgradeNodes(args common.Argument) error { - var loaderType string - - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewUpgradeNodesPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/os/config.go b/cmd/kk/pkg/phase/os/config.go deleted file mode 100755 index 907d38ac..00000000 --- a/cmd/kk/pkg/phase/os/config.go +++ /dev/null @@ -1,71 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package os - -import ( - "errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" -) - -func NewConfigOSPipeline(runtime *common.KubeRuntime) error { - - m := []module.Module{ - &precheck.NodePreCheckModule{}, - &os.RepositoryModule{Skip: !runtime.Arg.InstallPackages}, - &os.ConfigureOSModule{Skip: runtime.Cluster.System.SkipConfigureOS}, - } - - p := pipeline.Pipeline{ - Name: "ConfigOSPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func ConfigOS(args common.Argument) error { - var loaderType string - - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewConfigOSPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/phase/precheck/modules.go b/cmd/kk/pkg/phase/precheck/modules.go deleted file mode 100755 index 1e75a5f5..00000000 --- a/cmd/kk/pkg/phase/precheck/modules.go +++ /dev/null @@ -1,186 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package precheck - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" -) - -type UpgradePreCheckModule struct { - common.KubeModule -} - -func (c *UpgradePreCheckModule) Init() { - c.Name = "UpgradePreCheckModule" - c.Desc = "Do pre-check on for upgrade phase" - - nodePreCheck := &task.RemoteTask{ - Name: "NodePreCheck", - Desc: "A pre-check on nodes", - Hosts: c.Runtime.GetAllHosts(), - Action: new(precheck.NodePreCheck), - Parallel: true, - } - - getKubeConfig := &task.RemoteTask{ - Name: "GetKubeConfig", - Desc: "Get KubeConfig file", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(precheck.GetKubeConfig), - Parallel: true, - } - - getAllNodesK8sVersion := &task.RemoteTask{ - Name: "GetAllNodesK8sVersion", - Desc: "Get all nodes Kubernetes version", - Hosts: c.Runtime.GetHostsByRole(common.K8s), - Action: new(GetAllNodesK8sVersion), - Parallel: true, - } - - calculateMinK8sVersion := &task.LocalTask{ - Name: "CalculateMinK8sVersion", - Desc: "Calculate min Kubernetes version", - Action: new(precheck.CalculateMinK8sVersion), - } - - calculateMaxK8sVersion := &task.LocalTask{ - Name: "CalculateMaxK8sVersion", - Desc: "Calculate max Kubernetes version", - Action: new(CalculateMaxK8sVersion), - } - - checkDesiredK8sVersion := &task.LocalTask{ - Name: "CheckDesiredK8sVersion", - Desc: "Check desired Kubernetes version", - Action: new(precheck.CheckDesiredK8sVersion), - } - - checkUpgradeK8sVersion := &task.LocalTask{ - Name: "checkUpgradeK8sVersion", - Desc: "Check the Kubernetes version can correctly upgrade", - Action: new(CheckUpgradeK8sVersion), - } - - ksVersionCheck := &task.RemoteTask{ - Name: "KsVersionCheck", - Desc: "Check KubeSphere version", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(precheck.KsVersionCheck), - Parallel: true, - } - - getKubernetesNodesStatus := &task.RemoteTask{ - Name: "GetKubernetesNodesStatus", - Desc: "Get kubernetes nodes status", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(precheck.GetKubernetesNodesStatus), - Parallel: true, - } - - c.Tasks = []task.Interface{ - nodePreCheck, - getKubeConfig, - getAllNodesK8sVersion, - calculateMinK8sVersion, - calculateMaxK8sVersion, - checkDesiredK8sVersion, - checkUpgradeK8sVersion, - ksVersionCheck, - getKubernetesNodesStatus, - } -} - -type UpgradeKubeSpherePreCheckModule struct { - common.KubeModule -} - -func (c *UpgradeKubeSpherePreCheckModule) Init() { - c.Name = "UpgradeKubeSpherePreCheckModule" - c.Desc = "Do pre-check on for upgrade kubesphere phase" - - nodePreCheck := &task.RemoteTask{ - Name: "NodePreCheck", - Desc: "A pre-check on nodes", - Hosts: c.Runtime.GetAllHosts(), - Action: new(precheck.NodePreCheck), - Parallel: true, - } - - getKubeConfig := &task.RemoteTask{ - Name: "GetKubeConfig", - Desc: "Get KubeConfig file", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(precheck.GetKubeConfig), - Parallel: true, - } - - getMasterK8sVersion := &task.RemoteTask{ - Name: "GetMasterK8sVersion", - Desc: "get the master Kubernetes version", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(GetMasterK8sVersion), - } - - ksVersionCheck := &task.RemoteTask{ - Name: "KsVersionCheck", - Desc: "Check KubeSphere version", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(precheck.KsVersionCheck), - Parallel: true, - } - - c.Tasks = []task.Interface{ - nodePreCheck, - getKubeConfig, - getMasterK8sVersion, - ksVersionCheck, - } -} - -type UpgradeksPhaseDependencyCheckModule struct { - common.KubeModule -} - -func (c *UpgradeksPhaseDependencyCheckModule) Init() { - c.Name = "UpgradeksPhaseDependencyCheckModule" - c.Desc = "Check dependency matrix for KubeSphere and Kubernetes in ks phase" - - ksPhaseDependencyCheck := &task.RemoteTask{ - Name: "ksPhaseDependencyCheck", - Desc: "Check dependency matrix for KubeSphere and Kubernetes in ks phase", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(precheck.KubeSphereExist), - }, - Action: new(KsPhaseDependencyCheck), - } - - c.Tasks = []task.Interface{ - ksPhaseDependencyCheck, - } -} diff --git a/cmd/kk/pkg/phase/precheck/task.go b/cmd/kk/pkg/phase/precheck/task.go deleted file mode 100644 index 603956cb..00000000 --- a/cmd/kk/pkg/phase/precheck/task.go +++ /dev/null @@ -1,188 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package precheck - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere" -) - -type CheckUpgradeK8sVersion struct { - common.KubeAction -} - -func compareVersionsForUpgrade(currentVersion *versionutil.Version, currentMaxVersion *versionutil.Version, desiredVersion *versionutil.Version) error { - if desiredVersion.LessThan(currentMaxVersion) { - return errors.New("desired version to upgrade is less than the max version in cluster") - } - if desiredVersion.Major()-currentVersion.Major() != 0 || desiredVersion.Minor()-currentVersion.Minor() > 1 { - return errors.New("skipping MINOR versions when upgrading is unsupported") - } - return nil -} - -func (k *CheckUpgradeK8sVersion) Execute(_ connector.Runtime) error { - minK8sVersion, ok := k.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return errors.New("get current k8s version failed by pipeline cache") - } - minVersionObj, err := versionutil.ParseSemantic(minK8sVersion) - if err != nil { - return errors.Wrap(err, "parse min k8s version failed") - } - - maxK8sVersion, ok := k.PipelineCache.GetMustString(common.MaxK8sVersion) - if !ok { - return errors.New("get max k8s version failed by pipeline cache") - } - maxVersionObj, err := versionutil.ParseSemantic(maxK8sVersion) - if err != nil { - return errors.Wrap(err, "parse max k8s version failed") - } - - desiredVersion, ok := k.PipelineCache.GetMustString(common.DesiredK8sVersion) - if !ok { - return errors.New("get desired k8s version failed by pipeline cache") - } - desiredVersionObj, err := versionutil.ParseSemantic(desiredVersion) - if err != nil { - return errors.Wrap(err, "parse desired k8s version failed") - } - - if err := compareVersionsForUpgrade(minVersionObj, maxVersionObj, desiredVersionObj); err != nil { - return err - } - - k.PipelineCache.Set(common.PlanK8sVersion, desiredVersion) - return nil -} - -type CalculateMaxK8sVersion struct { - common.KubeAction -} - -func (g *CalculateMaxK8sVersion) Execute(runtime connector.Runtime) error { - versionList := make([]*versionutil.Version, 0, len(runtime.GetHostsByRole(common.K8s))) - for _, host := range runtime.GetHostsByRole(common.K8s) { - version, ok := host.GetCache().GetMustString(common.NodeK8sVersion) - if !ok { - return errors.Errorf("get node %s Kubernetes version failed by host cache", host.GetName()) - } - if versionObj, err := versionutil.ParseSemantic(version); err != nil { - return errors.Wrap(err, "parse node version failed") - } else { - versionList = append(versionList, versionObj) - } - } - - maxVersion := versionList[0] - for _, version := range versionList { - if maxVersion.LessThan(version) { - maxVersion = version - } - } - g.PipelineCache.Set(common.MaxK8sVersion, fmt.Sprintf("v%s", maxVersion)) - return nil -} - -type GetAllNodesK8sVersion struct { - common.KubeAction -} - -func (g *GetAllNodesK8sVersion) Execute(runtime connector.Runtime) error { - var nodeK8sVersion string - host := runtime.RemoteHost() - hostName := host.GetName() - - kubectlCmd := fmt.Sprintf("/usr/local/bin/kubectl get nodes %s", hostName) - kubectlVersionInfo, err := runtime.GetRunner().SudoCmd(kubectlCmd, false) - if err != nil { - errorMsg := fmt.Sprintf("get current version of %s failed", hostName) - return errors.Wrap(err, errorMsg) - } - - kubectlCmdGroup := strings.Split(kubectlVersionInfo, " ") - nodeK8sVersion = kubectlCmdGroup[len(kubectlCmdGroup)-1] - - if host.IsRole(common.Master) { - apiserverVersion, err := runtime.GetRunner().SudoCmd( - "cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep 'image:' | rev | cut -d ':' -f1 | rev", - false) - if err != nil { - return errors.Wrap(err, "get current kube-apiserver version failed") - } - nodeK8sVersion = apiserverVersion - } - host.GetCache().Set(common.NodeK8sVersion, nodeK8sVersion) - return nil -} - -type GetMasterK8sVersion struct { - common.KubeAction -} - -func (g *GetMasterK8sVersion) Execute(runtime connector.Runtime) error { - apiserverVersion, err := runtime.GetRunner().SudoCmd( - "cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep 'image:' | rev | cut -d ':' -f1 | rev", - false) - if err != nil { - return errors.Wrap(err, "get current kube-apiserver version failed") - } - g.PipelineCache.Set(common.K8sVersion, apiserverVersion) - return nil -} - -type KsPhaseDependencyCheck struct { - common.KubeAction -} - -func (d *KsPhaseDependencyCheck) Execute(_ connector.Runtime) error { - currentKsVersion, ok := d.PipelineCache.GetMustString(common.KubeSphereVersion) - if !ok { - return errors.New("get current KubeSphere version failed by pipeline cache") - } - currentK8sVersion, ok := d.PipelineCache.GetMustString(common.K8sVersion) - if !ok { - return errors.New("get current K8s version failed by pipeline cache") - } - desiredVersion := d.KubeConf.Cluster.KubeSphere.Version - - if d.KubeConf.Cluster.KubeSphere.Enabled { - KsInstaller, ok := kubesphere.VersionMap[desiredVersion] - if !ok { - return errors.New(fmt.Sprintf("Unsupported version: %s", desiredVersion)) - } - if currentKsVersion != desiredVersion { - if ok := KsInstaller.UpgradeSupport(currentKsVersion); !ok { - return errors.New(fmt.Sprintf("Unsupported upgrade plan: %s to %s", currentKsVersion, desiredVersion)) - } - } - if ok := KsInstaller.K8sSupport(currentK8sVersion); !ok { - return errors.New(fmt.Sprintf("KubeSphere %s does not support running on Kubernetes %s", - currentK8sVersion, d.KubeConf.Cluster.Kubernetes.Version)) - } - } - - return nil -} diff --git a/cmd/kk/pkg/pipelines/add_nodes.go b/cmd/kk/pkg/pipelines/add_nodes.go deleted file mode 100644 index 61dd067e..00000000 --- a/cmd/kk/pkg/pipelines/add_nodes.go +++ /dev/null @@ -1,199 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "fmt" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/customscripts" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/etcd" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k3s" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k8e" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/loadbalancer" -) - -func NewAddNodesPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - - m := []module.Module{ - &precheck.GreetingsModule{}, - &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, - &precheck.NodePreCheckModule{}, - &confirm.InstallConfirmModule{}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, - &binaries.NodeBinariesModule{}, - &os.ConfigureOSModule{Skip: runtime.Cluster.System.SkipConfigureOS}, - ®istry.RegistryCertsModule{Skip: len(runtime.GetHostsByRole(common.Registry)) == 0}, - //for one master to multi master kube-vip - &loadbalancer.KubevipModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - &kubernetes.RestartKubeletModule{}, - &kubernetes.StatusModule{}, - &container.InstallContainerModule{}, - &images.PullModule{Skip: runtime.Arg.SkipPullImages}, - &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &kubernetes.InstallKubeBinariesModule{}, - &kubernetes.JoinNodesModule{}, - &loadbalancer.HaproxyModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, - &kubernetes.ConfigureKubernetesModule{}, - &filesystem.ChownModule{}, - &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, - &customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall}, - } - - p := pipeline.Pipeline{ - Name: "AddNodesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - - return nil -} - -func NewK3sAddNodesPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - - m := []module.Module{ - &precheck.GreetingsModule{}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, - &binaries.K3sNodeBinariesModule{}, - &os.ConfigureOSModule{Skip: runtime.Cluster.System.SkipConfigureOS}, - &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, - &k3s.StatusModule{}, - &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &k3s.InstallKubeBinariesModule{}, - &k3s.JoinNodesModule{}, - &loadbalancer.K3sHaproxyModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, - &kubernetes.ConfigureKubernetesModule{}, - &filesystem.ChownModule{}, - &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, - &customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall}, - } - - p := pipeline.Pipeline{ - Name: "AddNodesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - - return nil -} - -func NewK8eAddNodesPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - - m := []module.Module{ - &precheck.GreetingsModule{}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, - &binaries.K8eNodeBinariesModule{}, - &os.ConfigureOSModule{Skip: runtime.Cluster.System.SkipConfigureOS}, - - &k8e.StatusModule{}, - &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &k8e.InstallKubeBinariesModule{}, - &k8e.JoinNodesModule{}, - &loadbalancer.K3sHaproxyModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, - &kubernetes.ConfigureKubernetesModule{}, - &filesystem.ChownModule{}, - &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, - } - - p := pipeline.Pipeline{ - Name: "AddNodesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - - return nil -} - -func AddNodes(args common.Argument, downloadCmd string) error { - args.DownloadCommand = func(path, url string) string { - // this is an extension point for downloading tools, for example users can set the timeout, proxy or retry under - // some poor network environment. Or users even can choose another cli, it might be wget. - // perhaps we should have a build-in download function instead of totally rely on the external one - return fmt.Sprintf(downloadCmd, path, url) - } - - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.K3s: - if err := NewK3sAddNodesPipeline(runtime); err != nil { - return err - } - case common.K8e: - if err := NewK8eAddNodesPipeline(runtime); err != nil { - return err - } - case common.Kubernetes: - fallthrough - default: - if err := NewAddNodesPipeline(runtime); err != nil { - return err - } - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/artifact_export.go b/cmd/kk/pkg/pipelines/artifact_export.go deleted file mode 100644 index 6e730deb..00000000 --- a/cmd/kk/pkg/pipelines/artifact_export.go +++ /dev/null @@ -1,148 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "fmt" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" -) - -func NewArtifactExportPipeline(runtime *common.ArtifactRuntime) error { - m := []module.Module{ - &confirm.CheckFileExistModule{FileName: runtime.Arg.Output}, - &images.CopyImagesToLocalModule{}, - &binaries.ArtifactBinariesModule{}, - &artifact.RepositoryModule{}, - &artifact.ArchiveModule{}, - &filesystem.ChownOutputModule{}, - &filesystem.ChownWorkDirModule{}, - } - - p := pipeline.Pipeline{ - Name: "ArtifactExportPipeline", - Modules: m, - Runtime: runtime, - ModulePostHooks: nil, - } - if err := p.Start(); err != nil { - return err - } - - return nil -} - -func NewK3sArtifactExportPipeline(runtime *common.ArtifactRuntime) error { - m := []module.Module{ - &confirm.CheckFileExistModule{FileName: runtime.Arg.Output}, - &images.CopyImagesToLocalModule{}, - &binaries.K3sArtifactBinariesModule{}, - &artifact.RepositoryModule{}, - &artifact.ArchiveModule{}, - &filesystem.ChownOutputModule{}, - &filesystem.ChownWorkDirModule{}, - } - - p := pipeline.Pipeline{ - Name: "K3sArtifactExportPipeline", - Modules: m, - Runtime: runtime, - ModulePostHooks: nil, - } - if err := p.Start(); err != nil { - return err - } - - return nil -} - -func NewK8eArtifactExportPipeline(runtime *common.ArtifactRuntime) error { - m := []module.Module{ - &confirm.CheckFileExistModule{FileName: runtime.Arg.Output}, - &images.CopyImagesToLocalModule{}, - &binaries.K8eArtifactBinariesModule{}, - &artifact.RepositoryModule{}, - &artifact.ArchiveModule{}, - &filesystem.ChownOutputModule{}, - &filesystem.ChownWorkDirModule{}, - } - - p := pipeline.Pipeline{ - Name: "K8eArtifactBinariesModule", - Modules: m, - Runtime: runtime, - ModulePostHooks: nil, - } - if err := p.Start(); err != nil { - return err - } - - return nil -} - -func ArtifactExport(args common.ArtifactArgument, downloadCmd string) error { - args.DownloadCommand = func(path, url string) string { - // this is an extension point for downloading tools, for example users can set the timeout, proxy or retry under - // some poor network environment. Or users even can choose another cli, it might be wget. - // perhaps we should have a build-in download function instead of totally rely on the external one - return fmt.Sprintf(downloadCmd, path, url) - } - - runtime, err := common.NewArtifactRuntime(args) - if err != nil { - return err - } - - if len(runtime.Spec.KubernetesDistributions) == 0 { - return errors.New("the length of kubernetes distributions can't be 0") - } - - pre := runtime.Spec.KubernetesDistributions[0].Type - for _, t := range runtime.Spec.KubernetesDistributions { - if t.Type != pre { - return errors.New("all the types of kubernetes distributions can't be different") - } - } - - switch runtime.Spec.KubernetesDistributions[0].Type { - case common.K3s: - if err := NewK3sArtifactExportPipeline(runtime); err != nil { - return err - } - case common.K8e: - if err := NewK8eArtifactExportPipeline(runtime); err != nil { - return err - } - case common.Kubernetes: - fallthrough - default: - if err := NewArtifactExportPipeline(runtime); err != nil { - return err - } - } - - return nil -} diff --git a/cmd/kk/pkg/pipelines/check_certs.go b/cmd/kk/pkg/pipelines/check_certs.go deleted file mode 100644 index 261a26e4..00000000 --- a/cmd/kk/pkg/pipelines/check_certs.go +++ /dev/null @@ -1,62 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" -) - -func CheckCertsPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.GreetingsModule{}, - &certs.CheckCertsModule{}, - &certs.PrintClusterCertsModule{}, - } - - p := pipeline.Pipeline{ - Name: "CheckCertsPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func CheckCerts(args common.Argument) error { - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - if err := CheckCertsPipeline(runtime); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/create_cluster.go b/cmd/kk/pkg/pipelines/create_cluster.go deleted file mode 100644 index 99fc3e51..00000000 --- a/cmd/kk/pkg/pipelines/create_cluster.go +++ /dev/null @@ -1,318 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "fmt" - - kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/addons" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/customscripts" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/etcd" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k3s" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k8e" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubesphere" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/loadbalancer" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/network" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/storage" -) - -func NewCreateClusterPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - skipPushImages := runtime.Arg.SkipPushImages || noArtifact || (!noArtifact && runtime.Cluster.Registry.PrivateRegistry == "") - skipLocalStorage := true - if runtime.Arg.DeployLocalStorage != nil { - skipLocalStorage = !*runtime.Arg.DeployLocalStorage - } else if runtime.Cluster.KubeSphere.Enabled { - skipLocalStorage = false - } - - m := []module.Module{ - &precheck.GreetingsModule{}, - &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, - &precheck.NodePreCheckModule{}, - &confirm.InstallConfirmModule{}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, - &binaries.NodeBinariesModule{}, - &os.ConfigureOSModule{Skip: runtime.Cluster.System.SkipConfigureOS}, - &kubernetes.StatusModule{}, - &container.InstallContainerModule{}, - &images.CopyImagesToRegistryModule{Skip: skipPushImages}, - &images.PullModule{Skip: runtime.Arg.SkipPullImages}, - &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &kubernetes.InstallKubeBinariesModule{}, - // init kubeVip on first master - &loadbalancer.KubevipModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - &kubernetes.InitKubernetesModule{}, - &dns.ClusterDNSModule{}, - &kubernetes.StatusModule{}, - &kubernetes.JoinNodesModule{}, - // deploy kubeVip on other masters - &loadbalancer.KubevipModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - &loadbalancer.HaproxyModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, - &network.DeployNetworkPluginModule{}, - &kubernetes.ConfigureKubernetesModule{}, - &filesystem.ChownModule{}, - &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, - &kubernetes.SecurityEnhancementModule{Skip: !runtime.Arg.SecurityEnhancement}, - &kubernetes.SaveKubeConfigModule{}, - &plugins.DeployPluginsModule{}, - &addons.AddonsModule{}, - &storage.DeployLocalVolumeModule{Skip: skipLocalStorage}, - &kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall}, - } - - p := pipeline.Pipeline{ - Name: "CreateClusterPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - - if runtime.Cluster.KubeSphere.Enabled { - - fmt.Print(`Installation is complete. - -Please check the result using the command: - - kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l 'app in (ks-install, ks-installer)' -o jsonpath='{.items[0].metadata.name}') -f - -`) - } else { - fmt.Print(`Installation is complete. - -Please check the result using the command: - - kubectl get pod -A - -`) - - } - - return nil -} - -func NewK3sCreateClusterPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - skipPushImages := runtime.Arg.SkipPushImages || noArtifact || (!noArtifact && runtime.Cluster.Registry.PrivateRegistry == "") - skipLocalStorage := true - if runtime.Arg.DeployLocalStorage != nil { - skipLocalStorage = !*runtime.Arg.DeployLocalStorage - } else if runtime.Cluster.KubeSphere.Enabled { - skipLocalStorage = false - } - - m := []module.Module{ - &precheck.GreetingsModule{}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, - &binaries.K3sNodeBinariesModule{}, - &os.ConfigureOSModule{Skip: runtime.Cluster.System.SkipConfigureOS}, - &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, - &k3s.StatusModule{}, - &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &loadbalancer.K3sKubevipModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - &k3s.InstallKubeBinariesModule{}, - &k3s.InitClusterModule{}, - &k3s.StatusModule{}, - &k3s.JoinNodesModule{}, - &images.CopyImagesToRegistryModule{Skip: skipPushImages}, - &loadbalancer.K3sHaproxyModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, - &network.DeployNetworkPluginModule{}, - &kubernetes.ConfigureKubernetesModule{}, - &filesystem.ChownModule{}, - &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, - &k3s.SaveKubeConfigModule{}, - &addons.AddonsModule{}, - &storage.DeployLocalVolumeModule{Skip: skipLocalStorage}, - &kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall}, - } - - p := pipeline.Pipeline{ - Name: "K3sCreateClusterPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - - if runtime.Cluster.KubeSphere.Enabled { - - fmt.Print(`Installation is complete. - -Please check the result using the command: - - kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l 'app in (ks-install, ks-installer)' -o jsonpath='{.items[0].metadata.name}') -f - -`) - } else { - fmt.Print(`Installation is complete. - -Please check the result using the command: - - kubectl get pod -A - -`) - - } - - return nil -} - -func NewK8eCreateClusterPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - skipPushImages := runtime.Arg.SkipPushImages || noArtifact || (!noArtifact && runtime.Cluster.Registry.PrivateRegistry == "") - skipLocalStorage := true - if runtime.Arg.DeployLocalStorage != nil { - skipLocalStorage = !*runtime.Arg.DeployLocalStorage - } else if runtime.Cluster.KubeSphere.Enabled { - skipLocalStorage = false - } - - m := []module.Module{ - &precheck.GreetingsModule{}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &os.RepositoryModule{Skip: noArtifact || !runtime.Arg.InstallPackages}, - &binaries.K8eNodeBinariesModule{}, - &os.ConfigureOSModule{Skip: runtime.Cluster.System.SkipConfigureOS}, - &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, - &k8e.StatusModule{}, - &etcd.PreCheckModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.CertsModule{}, - &etcd.InstallETCDBinaryModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.ConfigureModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &etcd.BackupModule{Skip: runtime.Cluster.Etcd.Type != kubekeyapiv1alpha2.KubeKey}, - &loadbalancer.K3sKubevipModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - &k8e.InstallKubeBinariesModule{}, - &k8e.InitClusterModule{}, - &k8e.StatusModule{}, - &k8e.JoinNodesModule{}, - &images.CopyImagesToRegistryModule{Skip: skipPushImages}, - &loadbalancer.K3sHaproxyModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, - &network.DeployNetworkPluginModule{}, - &kubernetes.ConfigureKubernetesModule{}, - &filesystem.ChownModule{}, - &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, - &k8e.SaveKubeConfigModule{}, - &addons.AddonsModule{}, - &storage.DeployLocalVolumeModule{Skip: skipLocalStorage}, - &kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &customscripts.CustomScriptsModule{Phase: "PostInstall", Scripts: runtime.Cluster.System.PostInstall}, - } - - p := pipeline.Pipeline{ - Name: "K8eCreateClusterPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - - if runtime.Cluster.KubeSphere.Enabled { - - fmt.Print(`Installation is complete. - -Please check the result using the command: - - kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l 'app in (ks-install, ks-installer)' -o jsonpath='{.items[0].metadata.name}') -f - -`) - } else { - fmt.Print(`Installation is complete. - -Please check the result using the command: - - kubectl get pod -A - -`) - - } - - return nil -} - -func CreateCluster(args common.Argument, downloadCmd string) error { - args.DownloadCommand = func(path, url string) string { - // this is an extension point for downloading tools, for example users can set the timeout, proxy or retry under - // some poor network environment. Or users even can choose another cli, it might be wget. - // perhaps we should have a build-in download function instead of totally rely on the external one - return fmt.Sprintf(downloadCmd, path, url) - } - - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.K3s: - if err := NewK3sCreateClusterPipeline(runtime); err != nil { - return err - } - case common.K8e: - if err := NewK8eCreateClusterPipeline(runtime); err != nil { - return err - } - case common.Kubernetes: - fallthrough - default: - if err := NewCreateClusterPipeline(runtime); err != nil { - return err - } - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/delete_cluster.go b/cmd/kk/pkg/pipelines/delete_cluster.go deleted file mode 100644 index e93def1f..00000000 --- a/cmd/kk/pkg/pipelines/delete_cluster.go +++ /dev/null @@ -1,127 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k3s" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/k8e" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/loadbalancer" -) - -func NewDeleteClusterPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.GreetingsModule{}, - &confirm.DeleteClusterConfirmModule{Skip: runtime.Arg.SkipConfirmCheck}, - &kubernetes.ResetClusterModule{}, - &container.UninstallContainerModule{Skip: !runtime.Arg.DeleteCRI}, - &os.ClearOSEnvironmentModule{}, - &certs.UninstallAutoRenewCertsModule{}, - &loadbalancer.DeleteVIPModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - } - - p := pipeline.Pipeline{ - Name: "DeleteClusterPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func NewK3sDeleteClusterPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.GreetingsModule{}, - &confirm.DeleteClusterConfirmModule{}, - &k3s.DeleteClusterModule{}, - &os.ClearOSEnvironmentModule{}, - &certs.UninstallAutoRenewCertsModule{}, - &loadbalancer.DeleteVIPModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - } - - p := pipeline.Pipeline{ - Name: "K3sDeleteClusterPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func NewK8eDeleteClusterPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.GreetingsModule{}, - &confirm.DeleteClusterConfirmModule{}, - &k8e.DeleteClusterModule{}, - &os.ClearOSEnvironmentModule{}, - &certs.UninstallAutoRenewCertsModule{}, - } - - p := pipeline.Pipeline{ - Name: "K8eDeleteClusterPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func DeleteCluster(args common.Argument) error { - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.K3s: - if err := NewK3sDeleteClusterPipeline(runtime); err != nil { - return err - } - case common.K8e: - if err := NewK8eDeleteClusterPipeline(runtime); err != nil { - return err - } - case common.Kubernetes: - fallthrough - default: - if err := NewDeleteClusterPipeline(runtime); err != nil { - return err - } - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/delete_node.go b/cmd/kk/pkg/pipelines/delete_node.go deleted file mode 100644 index e55ae7a9..00000000 --- a/cmd/kk/pkg/pipelines/delete_node.go +++ /dev/null @@ -1,68 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/loadbalancer" -) - -func DeleteNodePipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.GreetingsModule{}, - &confirm.DeleteNodeConfirmModule{Skip: runtime.Arg.SkipConfirmCheck}, - &kubernetes.CompareConfigAndClusterInfoModule{}, - &kubernetes.DeleteKubeNodeModule{}, - &os.ClearNodeOSModule{}, - &loadbalancer.DeleteVIPModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabledVip()}, - } - - p := pipeline.Pipeline{ - Name: "DeleteNodePipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func DeleteNode(args common.Argument) error { - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - if err := DeleteNodePipeline(runtime); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/init_dependencies.go b/cmd/kk/pkg/pipelines/init_dependencies.go deleted file mode 100644 index 0f8c6871..00000000 --- a/cmd/kk/pkg/pipelines/init_dependencies.go +++ /dev/null @@ -1,70 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/customscripts" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" -) - -func NewInitDependenciesPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - - m := []module.Module{ - &precheck.GreetingsModule{}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &os.RepositoryModule{Skip: noArtifact}, - &os.RepositoryOnlineModule{Skip: !noArtifact}, - &filesystem.ChownWorkDirModule{}, - &customscripts.CustomScriptsModule{Phase: "PreInstall", Scripts: runtime.Cluster.System.PreInstall}, - } - - p := pipeline.Pipeline{ - Name: "InitDependenciesPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func InitDependencies(args common.Argument) error { - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - if err := NewInitDependenciesPipeline(runtime); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/init_registry.go b/cmd/kk/pkg/pipelines/init_registry.go deleted file mode 100644 index 38c26c76..00000000 --- a/cmd/kk/pkg/pipelines/init_registry.go +++ /dev/null @@ -1,86 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "fmt" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" - "github.com/pkg/errors" -) - -func NewInitRegistryPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - - m := []module.Module{ - &precheck.GreetingsModule{}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &binaries.RegistryPackageModule{}, - &os.ConfigureOSModule{Skip: runtime.Cluster.System.SkipConfigureOS}, - ®istry.RegistryCertsModule{}, - ®istry.InstallRegistryModule{}, - &filesystem.ChownWorkDirModule{}, - } - - p := pipeline.Pipeline{ - Name: "InitRegistryPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func InitRegistry(args common.Argument, downloadCmd string) error { - args.DownloadCommand = func(path, url string) string { - // this is an extension point for downloading tools, for example users can set the timeout, proxy or retry under - // some poor network environment. Or users even can choose another cli, it might be wget. - // perhaps we should have a build-in download function instead of totally rely on the external one - return fmt.Sprintf(downloadCmd, path, url) - } - - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - if len(runtime.GetHostsByRole("registry")) <= 0 { - logger.Log.Fatal(errors.New("The number of registry must be greater then 0.")) - } - - if err := NewInitRegistryPipeline(runtime); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/migrate_container.go b/cmd/kk/pkg/pipelines/migrate_container.go deleted file mode 100644 index 0378e5ad..00000000 --- a/cmd/kk/pkg/pipelines/migrate_container.go +++ /dev/null @@ -1,68 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "fmt" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" -) - -func MigrateCriPipeline(runtime *common.KubeRuntime) error { - fmt.Println("MigrateContainerdPipeline called") - m := []module.Module{ - &precheck.GreetingsModule{}, - &confirm.MigrateCriConfirmModule{}, - &binaries.CriBinariesModule{}, - &container.CriMigrateModule{}, - } - p := pipeline.Pipeline{ - Name: "MigrateContainerdPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func MigrateCri(args common.Argument, downloadCmd string) error { - args.DownloadCommand = func(path, url string) string { - return fmt.Sprintf(downloadCmd, path, url) - } - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - if err := MigrateCriPipeline(runtime); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/renew_certs.go b/cmd/kk/pkg/pipelines/renew_certs.go deleted file mode 100644 index f2753eba..00000000 --- a/cmd/kk/pkg/pipelines/renew_certs.go +++ /dev/null @@ -1,63 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" -) - -func RenewCertsPipeline(runtime *common.KubeRuntime) error { - m := []module.Module{ - &precheck.GreetingsModule{}, - &certs.RenewCertsModule{}, - &certs.CheckCertsModule{}, - &certs.PrintClusterCertsModule{}, - } - - p := pipeline.Pipeline{ - Name: "RenewCertsPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func RenewCerts(args common.Argument) error { - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - if err := RenewCertsPipeline(runtime); err != nil { - return err - } - return nil -} diff --git a/cmd/kk/pkg/pipelines/upgrade_cluster.go b/cmd/kk/pkg/pipelines/upgrade_cluster.go deleted file mode 100644 index 8aafd4f0..00000000 --- a/cmd/kk/pkg/pipelines/upgrade_cluster.go +++ /dev/null @@ -1,100 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pipelines - -import ( - "fmt" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubesphere" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/loadbalancer" -) - -func NewUpgradeClusterPipeline(runtime *common.KubeRuntime) error { - noArtifact := runtime.Arg.Artifact == "" - - m := []module.Module{ - &precheck.GreetingsModule{}, - &precheck.NodePreCheckModule{}, - &precheck.ClusterPreCheckModule{}, - &confirm.UpgradeConfirmModule{Skip: runtime.Arg.SkipConfirmCheck}, - &artifact.UnArchiveModule{Skip: noArtifact}, - &kubernetes.SetUpgradePlanModule{Step: kubernetes.ToV121}, - &kubernetes.ProgressiveUpgradeModule{Step: kubernetes.ToV121}, - &loadbalancer.HaproxyModule{Skip: !runtime.Cluster.ControlPlaneEndpoint.IsInternalLBEnabled()}, - &kubesphere.CleanClusterConfigurationModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.ConvertModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.DeployModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubesphere.CheckResultModule{Skip: !runtime.Cluster.KubeSphere.Enabled}, - &kubernetes.SetUpgradePlanModule{Step: kubernetes.ToV122}, - &kubernetes.ProgressiveUpgradeModule{Step: kubernetes.ToV122}, - &filesystem.ChownModule{}, - &certs.AutoRenewCertsModule{Skip: !runtime.Cluster.Kubernetes.EnableAutoRenewCerts()}, - } - - p := pipeline.Pipeline{ - Name: "UpgradeClusterPipeline", - Modules: m, - Runtime: runtime, - } - if err := p.Start(); err != nil { - return err - } - return nil -} - -func UpgradeCluster(args common.Argument, downloadCmd string) error { - args.DownloadCommand = func(path, url string) string { - // this is an extension point for downloading tools, for example users can set the timeout, proxy or retry under - // some poor network environment. Or users even can choose another cli, it might be wget. - // perhaps we should have a build-in download function instead of totally rely on the external one - return fmt.Sprintf(downloadCmd, path, url) - } - - var loaderType string - if args.FilePath != "" { - loaderType = common.File - } else { - loaderType = common.AllInOne - } - - runtime, err := common.NewKubeRuntime(loaderType, args) - if err != nil { - return err - } - - switch runtime.Cluster.Kubernetes.Type { - case common.Kubernetes: - if err := NewUpgradeClusterPipeline(runtime); err != nil { - return err - } - default: - return errors.New("unsupported cluster kubernetes type") - } - - return nil -} diff --git a/cmd/kk/pkg/plugins/dns/module.go b/cmd/kk/pkg/plugins/dns/module.go deleted file mode 100644 index befb2baa..00000000 --- a/cmd/kk/pkg/plugins/dns/module.go +++ /dev/null @@ -1,162 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package dns - -import ( - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns/templates" -) - -type ClusterDNSModule struct { - common.KubeModule -} - -func (c *ClusterDNSModule) Init() { - c.Name = "ClusterDNSModule" - c.Desc = "Deploy cluster dns" - - generateCorednsConfigMap := &task.RemoteTask{ - Name: "GenerateCorednsConfigMap", - Desc: "Generate coredns configmap", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - }, - Action: &action.Template{ - Template: templates.CorednsConfigMap, - Dst: filepath.Join(common.KubeConfigDir, templates.CorednsConfigMap.Name()), - Data: util.Data{ - "DNSEtcHosts": c.KubeConf.Cluster.DNS.DNSEtcHosts, - "ExternalZones": c.KubeConf.Cluster.DNS.CoreDNS.ExternalZones, - "AdditionalConfigs": c.KubeConf.Cluster.DNS.CoreDNS.AdditionalConfigs, - "RewriteBlock": c.KubeConf.Cluster.DNS.CoreDNS.RewriteBlock, - "ClusterDomain": c.KubeConf.Cluster.Kubernetes.DNSDomain, - "UpstreamDNSServers": c.KubeConf.Cluster.DNS.CoreDNS.UpstreamDNSServers, - }, - }, - Parallel: true, - } - - applyCorednsConfigMap := &task.RemoteTask{ - Name: "ApplyCorednsConfigMap", - Desc: "Apply coredns configmap", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - }, - Action: new(ApplyCorednsConfigMap), - Parallel: true, - Retry: 5, - } - - generateCoreDNS := &task.RemoteTask{ - Name: "GenerateCoreDNS", - Desc: "Generate coredns manifests", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - }, - Action: new(GenerateCorednsmanifests), - Parallel: true, - } - - deployCoredns := &task.RemoteTask{ - Name: "DeployCoreDNS", - Desc: "Deploy coredns", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - }, - Action: new(DeployCoreDNS), - Parallel: true, - } - - generateNodeLocalDNSConfigMap := &task.RemoteTask{ - Name: "GenerateNodeLocalDNSConfigMap", - Desc: "Generate nodelocaldns configmap", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(EnableNodeLocalDNS), - }, - Action: new(GenerateNodeLocalDNSConfigMap), - Parallel: true, - } - - applyNodeLocalDNSConfigMap := &task.RemoteTask{ - Name: "ApplyNodeLocalDNSConfigMap", - Desc: "Apply nodelocaldns configmap", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(EnableNodeLocalDNS), - }, - Action: new(ApplyNodeLocalDNSConfigMap), - Parallel: true, - Retry: 5, - } - - generateNodeLocalDNS := &task.RemoteTask{ - Name: "GenerateNodeLocalDNS", - Desc: "Generate nodelocaldns", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(EnableNodeLocalDNS), - }, - Action: &action.Template{ - Template: templates.NodeLocalDNSService, - Dst: filepath.Join(common.KubeConfigDir, templates.NodeLocalDNSService.Name()), - Data: util.Data{ - "NodelocaldnsImage": images.GetImage(c.Runtime, c.KubeConf, "k8s-dns-node-cache").ImageName(), - "DNSEtcHosts": c.KubeConf.Cluster.DNS.DNSEtcHosts, - }, - }, - Parallel: true, - } - - applyNodeLocalDNS := &task.RemoteTask{ - Name: "DeployNodeLocalDNS", - Desc: "Deploy nodelocaldns", - Hosts: c.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(EnableNodeLocalDNS), - }, - Action: new(DeployNodeLocalDNS), - Parallel: true, - Retry: 5, - } - - c.Tasks = []task.Interface{ - generateCorednsConfigMap, - applyCorednsConfigMap, - generateCoreDNS, - deployCoredns, - generateNodeLocalDNSConfigMap, - applyNodeLocalDNSConfigMap, - generateNodeLocalDNS, - applyNodeLocalDNS, - } -} diff --git a/cmd/kk/pkg/plugins/dns/prepares.go b/cmd/kk/pkg/plugins/dns/prepares.go deleted file mode 100644 index 1b46cf55..00000000 --- a/cmd/kk/pkg/plugins/dns/prepares.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package dns - -import ( - "strings" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type CoreDNSExist struct { - common.KubePrepare - Not bool -} - -func (c *CoreDNSExist) PreCheck(runtime connector.Runtime) (bool, error) { - _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl get svc -n kube-system coredns", false) - if err != nil { - if strings.Contains(err.Error(), "NotFound") { - return c.Not, nil - } - return false, err - } - return !c.Not, nil -} - -type EnableNodeLocalDNS struct { - common.KubePrepare -} - -func (e *EnableNodeLocalDNS) PreCheck(runtime connector.Runtime) (bool, error) { - if e.KubeConf.Cluster.Kubernetes.EnableNodelocaldns() { - return true, nil - } - return false, nil -} - -type NodeLocalDNSConfigMapNotExist struct { - common.KubePrepare -} - -func (n *NodeLocalDNSConfigMapNotExist) PreCheck(runtime connector.Runtime) (bool, error) { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl get cm -n kube-system nodelocaldns", false); err != nil { - if strings.Contains(err.Error(), "NotFound") { - return true, nil - } - return false, err - } - return false, nil -} diff --git a/cmd/kk/pkg/plugins/dns/tasks.go b/cmd/kk/pkg/plugins/dns/tasks.go deleted file mode 100644 index 8a336d34..00000000 --- a/cmd/kk/pkg/plugins/dns/tasks.go +++ /dev/null @@ -1,133 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package dns - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/pkg/errors" - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns/templates" -) - -type GenerateCorednsmanifests struct { - common.KubeAction -} - -func (g *GenerateCorednsmanifests) Execute(runtime connector.Runtime) error { - templateAction := action.Template{ - Template: templates.Coredns, - Dst: filepath.Join(common.KubeConfigDir, templates.Coredns.Name()), - Data: util.Data{ - "ClusterIP": g.KubeConf.Cluster.CorednsClusterIP(), - "CorednsImage": images.GetImage(runtime, g.KubeConf, "coredns").ImageName(), - "DNSEtcHosts": g.KubeConf.Cluster.DNS.DNSEtcHosts, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type DeployCoreDNS struct { - common.KubeAction -} - -func (o *DeployCoreDNS) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl delete svc -n kube-system --field-selector metadata.name=kube-dns", true); err != nil { - return errors.Wrap(errors.WithStack(err), "remove old coredns svc") - } - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/coredns.yaml", true); err != nil { - return errors.Wrap(errors.WithStack(err), "update coredns failed") - } - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl -n kube-system rollout restart deploy coredns", true); err != nil { - return errors.Wrap(errors.WithStack(err), "restart coredns failed") - } - return nil -} - -type ApplyCorednsConfigMap struct { - common.KubeAction -} - -func (o *ApplyCorednsConfigMap) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/coredns-configmap.yaml", true); err != nil { - return errors.Wrap(errors.WithStack(err), "create coredns configmap failed") - } - return nil -} - -type DeployNodeLocalDNS struct { - common.KubeAction -} - -func (d *DeployNodeLocalDNS) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/nodelocaldns.yaml", true); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy nodelocaldns failed") - } - return nil -} - -type GenerateNodeLocalDNSConfigMap struct { - common.KubeAction -} - -func (g *GenerateNodeLocalDNSConfigMap) Execute(runtime connector.Runtime) error { - clusterIP, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl get svc -n kube-system coredns -o jsonpath='{.spec.clusterIP}'", false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get clusterIP failed") - } - - if len(clusterIP) == 0 { - clusterIP = g.KubeConf.Cluster.CorednsClusterIP() - } - - templateAction := action.Template{ - Template: templates.NodeLocalDNSConfigMap, - Dst: filepath.Join(common.KubeConfigDir, templates.NodeLocalDNSConfigMap.Name()), - Data: util.Data{ - "ForwardTarget": clusterIP, - "DNSDomain": g.KubeConf.Cluster.Kubernetes.DNSDomain, - "ExternalZones": g.KubeConf.Cluster.DNS.NodeLocalDNS.ExternalZones, - "DNSEtcHosts": g.KubeConf.Cluster.DNS.DNSEtcHosts, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - return nil -} - -type ApplyNodeLocalDNSConfigMap struct { - common.KubeAction -} - -func (a *ApplyNodeLocalDNSConfigMap) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/nodelocaldns-configmap.yaml", true); err != nil { - return errors.Wrap(errors.WithStack(err), "apply nodelocaldns configmap failed") - } - return nil -} diff --git a/cmd/kk/pkg/plugins/dns/templates/coredns.go b/cmd/kk/pkg/plugins/dns/templates/coredns.go deleted file mode 100644 index 4443c153..00000000 --- a/cmd/kk/pkg/plugins/dns/templates/coredns.go +++ /dev/null @@ -1,211 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var ( - Coredns = template.Must(template.New("coredns.yaml").Parse( - dedent.Dedent(`--- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - kubernetes.io/bootstrapping: rbac-defaults - addonmanager.kubernetes.io/mode: Reconcile - name: system:coredns -rules: -- apiGroups: - - "" - resources: - - endpoints - - services - - pods - - namespaces - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - ---- -apiVersion: v1 -kind: Service -metadata: - name: coredns - namespace: kube-system - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "CoreDNS" - addonmanager.kubernetes.io/mode: Reconcile - annotations: - prometheus.io/port: "9153" - prometheus.io/scrape: "true" - createdby: 'kubekey' -spec: - selector: - k8s-app: kube-dns - clusterIP: {{ .ClusterIP }} - ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP - - name: metrics - port: 9153 - protocol: TCP - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: "coredns" - namespace: kube-system - labels: - k8s-app: "kube-dns" - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/name: "coredns" -spec: - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 0 - maxSurge: 10% - selector: - matchLabels: - k8s-app: kube-dns - template: - metadata: - labels: - k8s-app: kube-dns - annotations: - createdby: 'kubekey' - spec: - securityContext: - seccompProfile: - type: RuntimeDefault - priorityClassName: system-cluster-critical - serviceAccountName: coredns - nodeSelector: - kubernetes.io/os: linux - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchLabels: - k8s-app: kube-dns - topologyKey: "kubernetes.io/hostname" - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: In - values: - - "" - containers: - - name: coredns - image: "{{ .CorednsImage }}" - imagePullPolicy: IfNotPresent - resources: - # TODO: Set memory limits when we've profiled the container for large - # clusters, then set request = limit to keep this container in - # guaranteed class. Currently, this container falls into the - # "burstable" category so the kubelet doesn't backoff from restarting it. - limits: - memory: 300Mi - requests: - cpu: 100m - memory: 70Mi - args: [ "-conf", "/etc/coredns/Corefile" ] - volumeMounts: - - name: config-volume - mountPath: /etc/coredns - ports: - - containerPort: 53 - name: dns - protocol: UDP - - containerPort: 53 - name: dns-tcp - protocol: TCP - - containerPort: 9153 - name: metrics - protocol: TCP - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - all - readOnlyRootFilesystem: true - livenessProbe: - httpGet: - path: /health - port: 8080 - scheme: HTTP - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 10 - readinessProbe: - httpGet: - path: /ready - port: 8181 - scheme: HTTP - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 10 - dnsPolicy: Default - volumes: - - name: config-volume - configMap: - name: coredns - items: - - key: Corefile - path: Corefile -{{ if .DNSEtcHosts }} - - key: hosts - path: hosts -{{ end }} - - `))) -) diff --git a/cmd/kk/pkg/plugins/dns/templates/coredns_configmap.go b/cmd/kk/pkg/plugins/dns/templates/coredns_configmap.go deleted file mode 100644 index fde5f4c3..00000000 --- a/cmd/kk/pkg/plugins/dns/templates/coredns_configmap.go +++ /dev/null @@ -1,97 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" - "github.com/lithammer/dedent" - "text/template" -) - -var ( - CorednsConfigMap = template.Must(template.New("coredns-configmap.yaml").Funcs(utils.FuncMap).Parse( - dedent.Dedent(`--- -apiVersion: v1 -kind: ConfigMap -metadata: - name: coredns - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: EnsureExists -data: - Corefile: | -{{- if .ExternalZones }} -{{- range .ExternalZones }} -{{ range .Zones }}{{ . | indent 4 }} {{ end }}{ - log - errors -{{- if .Rewrite }} -{{- range .Rewrite }} - rewrite {{ . }} -{{- end }} -{{- end }} - forward .{{ range .Nameservers }} {{ . }}{{ end}} - loadbalance - cache {{ .Cache }} - reload -{{- if $.DNSEtcHosts }} - hosts /etc/coredns/hosts { - fallthrough - } -{{- end }} - } -{{- end }} -{{- end }} - .:53 { -{{- if .AdditionalConfigs }} -{{ .AdditionalConfigs | indent 8 }} -{{- end }} - errors - health { - lameduck 5s - } -{{- if .RewriteBlock }} -{{ .RewriteBlock | indent 8 }} -{{- end }} - ready - kubernetes {{ .ClusterDomain }} in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 30 - } - prometheus :9153 - forward . {{ if .UpstreamDNSServers }}{{ range .UpstreamDNSServers }}{{ . }} {{ end }}{{else}}/etc/resolv.conf{{ end }} { - prefer_udp - max_concurrent 1000 - } - cache 30 - loop - reload - loadbalance -{{- if .DNSEtcHosts }} - hosts /etc/coredns/hosts { - fallthrough - } -{{- end }} - } -{{- if .DNSEtcHosts }} - hosts: | -{{ .DNSEtcHosts | indent 4 }} -{{- end }} - - `))) -) diff --git a/cmd/kk/pkg/plugins/dns/templates/nodelocaldns.go b/cmd/kk/pkg/plugins/dns/templates/nodelocaldns.go deleted file mode 100644 index 6d64ebda..00000000 --- a/cmd/kk/pkg/plugins/dns/templates/nodelocaldns.go +++ /dev/null @@ -1,136 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var NodeLocalDNSService = template.Must(template.New("nodelocaldns.yaml").Parse( - dedent.Dedent(`--- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nodelocaldns - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: Reconcile - ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: nodelocaldns - namespace: kube-system - labels: - k8s-app: kube-dns - addonmanager.kubernetes.io/mode: Reconcile -spec: - selector: - matchLabels: - k8s-app: nodelocaldns - template: - metadata: - labels: - k8s-app: nodelocaldns - annotations: - prometheus.io/scrape: 'true' - prometheus.io/port: '9253' - spec: - nodeSelector: - kubernetes.io/os: linux - priorityClassName: system-cluster-critical - serviceAccountName: nodelocaldns - hostNetwork: true - dnsPolicy: Default # Don't use cluster DNS. - tolerations: - - effect: NoSchedule - operator: "Exists" - - effect: NoExecute - operator: "Exists" - - key: "CriticalAddonsOnly" - operator: "Exists" - containers: - - name: node-cache - image: {{ .NodelocaldnsImage }} - resources: - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 70Mi - args: [ "-localip", "169.254.25.10", "-conf", "/etc/coredns/Corefile", "-upstreamsvc", "coredns" ] - securityContext: - privileged: true - ports: - - containerPort: 53 - name: dns - protocol: UDP - - containerPort: 53 - name: dns-tcp - protocol: TCP - - containerPort: 9253 - name: metrics - protocol: TCP - livenessProbe: - httpGet: - host: 169.254.25.10 - path: /health - port: 9254 - scheme: HTTP - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 10 - readinessProbe: - httpGet: - host: 169.254.25.10 - path: /health - port: 9254 - scheme: HTTP - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 10 - volumeMounts: - - name: config-volume - mountPath: /etc/coredns - - name: xtables-lock - mountPath: /run/xtables.lock - volumes: - - name: config-volume - configMap: - name: nodelocaldns - items: - - key: Corefile - path: Corefile -{{- if .DNSEtcHosts }} - - key: hosts - path: hosts -{{ end }} - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force - # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. - terminationGracePeriodSeconds: 0 - updateStrategy: - rollingUpdate: - maxUnavailable: 20% - type: RollingUpdate - - `))) diff --git a/cmd/kk/pkg/plugins/dns/templates/nodelocaldns_configmap.go b/cmd/kk/pkg/plugins/dns/templates/nodelocaldns_configmap.go deleted file mode 100644 index 55611500..00000000 --- a/cmd/kk/pkg/plugins/dns/templates/nodelocaldns_configmap.go +++ /dev/null @@ -1,116 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" - "github.com/lithammer/dedent" - "text/template" -) - -var NodeLocalDNSConfigMap = template.Must(template.New("nodelocaldns-configmap.yaml").Funcs(utils.FuncMap).Parse( - dedent.Dedent(`--- -apiVersion: v1 -kind: ConfigMap -metadata: - name: nodelocaldns - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: EnsureExists - -data: - Corefile: | -{{- if .ExternalZones }} -{{- range .ExternalZones }} -{{ range .Zones }}{{ . | indent 4 }} {{ end}} { - errors - cache {{ .Cache }} - reload -{{- if .Rewrite }} -{{- range .Rewrite }} - rewrite {{ . }} -{{- end }} -{{- end }} - loop - bind 169.254.25.10 - forward . {{ range .Nameservers }} {{ . }}{{ end }} - prometheus :9253 - log -{{- if $.DNSEtcHosts }} - hosts /etc/coredns/hosts { - fallthrough - } -{{- end }} - } -{{- end }} -{{- end }} - {{ .DNSDomain }}:53 { - errors - cache { - success 9984 30 - denial 9984 5 - } - reload - loop - bind 169.254.25.10 - forward . {{ .ForwardTarget }} { - force_tcp - } - prometheus :9253 - health 169.254.25.10:9254 - } - in-addr.arpa:53 { - errors - cache 30 - reload - loop - bind 169.254.25.10 - forward . {{ .ForwardTarget }} { - force_tcp - } - prometheus :9253 - } - ip6.arpa:53 { - errors - cache 30 - reload - loop - bind 169.254.25.10 - forward . {{ .ForwardTarget }} { - force_tcp - } - prometheus :9253 - } - .:53 { - errors - cache 30 - reload - loop - bind 169.254.25.10 - forward . /etc/resolv.conf - prometheus :9253 -{{- if .DNSEtcHosts }} - hosts /etc/coredns/hosts { - fallthrough - } -{{- end }} - } -{{- if .DNSEtcHosts }} - hosts: | -{{ .DNSEtcHosts | indent 4}} -{{- end }} -`))) diff --git a/cmd/kk/pkg/plugins/kata.go b/cmd/kk/pkg/plugins/kata.go deleted file mode 100644 index 3d0b4776..00000000 --- a/cmd/kk/pkg/plugins/kata.go +++ /dev/null @@ -1,206 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package plugins - -import ( - "path/filepath" - "text/template" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" -) - -// Kata Containers is an open source community working to build a secure container runtime with lightweight virtual -// machines that feel and perform like containers, but provide stronger workload isolation using hardware virtualization -// technology as a second layer of defense. - -var ( - KataDeploy = template.Must(template.New("kata-deploy.yaml").Parse( - dedent.Dedent(`--- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kata-label-node - namespace: kube-system ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: node-labeler -rules: -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "patch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: kata-label-node-rb -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: node-labeler -subjects: -- kind: ServiceAccount - name: kata-label-node - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kata-deploy - namespace: kube-system -spec: - selector: - matchLabels: - name: kata-deploy - template: - metadata: - labels: - name: kata-deploy - spec: - serviceAccountName: kata-label-node - containers: - - name: kube-kata - image: {{ .KataDeployImage }} - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - exec: - command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"] - command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh install" ] - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - privileged: false - volumeMounts: - - name: crio-conf - mountPath: /etc/crio/ - - name: containerd-conf - mountPath: /etc/containerd/ - - name: kata-artifacts - mountPath: /opt/kata/ - - name: dbus - mountPath: /var/run/dbus - - name: systemd - mountPath: /run/systemd - - name: local-bin - mountPath: /usr/local/bin/ - volumes: - - name: crio-conf - hostPath: - path: /etc/crio/ - - name: containerd-conf - hostPath: - path: /etc/containerd/ - - name: kata-artifacts - hostPath: - path: /opt/kata/ - type: DirectoryOrCreate - - name: dbus - hostPath: - path: /var/run/dbus - - name: systemd - hostPath: - path: /run/systemd - - name: local-bin - hostPath: - path: /usr/local/bin/ - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate ---- -kind: RuntimeClass -apiVersion: node.k8s.io/v1beta1 -metadata: - name: kata-qemu -handler: kata-qemu -overhead: - podFixed: - memory: "160Mi" - cpu: "250m" ---- -kind: RuntimeClass -apiVersion: node.k8s.io/v1beta1 -metadata: - name: kata-clh -handler: kata-clh -overhead: - podFixed: - memory: "130Mi" - cpu: "250m" ---- -kind: RuntimeClass -apiVersion: node.k8s.io/v1beta1 -metadata: - name: kata-fc -handler: kata-fc -overhead: - podFixed: - memory: "130Mi" - cpu: "250m" - `))) -) - -func DeployKataTasks(d *DeployPluginsModule) []task.Interface { - generateKataDeployManifests := &task.RemoteTask{ - Name: "GenerateKataDeployManifests", - Desc: "Generate kata-deploy manifests", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: &action.Template{ - Template: KataDeploy, - Data: util.Data{ - "KataDeployImage": images.GetImage(d.Runtime, d.KubeConf, "kata-deploy").ImageName(), - }, - Dst: filepath.Join(common.KubeAddonsDir, KataDeploy.Name()), - }, - Parallel: false, - } - - deployKata := &task.RemoteTask{ - Name: "ApplyKataDeployManifests", - Desc: "Apply kata-deploy manifests", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(ApplyKataDeployManifests), - } - - return []task.Interface{ - generateKataDeployManifests, - deployKata, - } -} - -type ApplyKataDeployManifests struct { - common.KubeAction -} - -func (a *ApplyKataDeployManifests) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/addons/kata-deploy.yaml", true); err != nil { - return errors.Wrap(errors.WithStack(err), "apply kata-deploy manifests failed") - } - return nil -} diff --git a/cmd/kk/pkg/plugins/modules.go b/cmd/kk/pkg/plugins/modules.go deleted file mode 100644 index f69800fc..00000000 --- a/cmd/kk/pkg/plugins/modules.go +++ /dev/null @@ -1,35 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package plugins - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" -) - -type DeployPluginsModule struct { - common.KubeModule -} - -func (d *DeployPluginsModule) Init() { - d.Name = "DeployPluginsModule" - d.Desc = "Deploy plugins for cluster" - - if d.KubeConf.Cluster.Kubernetes.EnableKataDeploy() && (d.KubeConf.Cluster.Kubernetes.ContainerManager == common.Containerd || d.KubeConf.Cluster.Kubernetes.ContainerManager == common.Crio) { - d.Tasks = append(d.Tasks, DeployKataTasks(d)...) - } - - if d.KubeConf.Cluster.Kubernetes.EnableNodeFeatureDiscovery() { - d.Tasks = append(d.Tasks, DeployNodeFeatureDiscoveryTasks(d)...) - } -} diff --git a/cmd/kk/pkg/plugins/network/cilium-1.11.7.tgz b/cmd/kk/pkg/plugins/network/cilium-1.11.7.tgz deleted file mode 100644 index 7effa6f3..00000000 Binary files a/cmd/kk/pkg/plugins/network/cilium-1.11.7.tgz and /dev/null differ diff --git a/cmd/kk/pkg/plugins/network/hybridnet-0.6.6.tgz b/cmd/kk/pkg/plugins/network/hybridnet-0.6.6.tgz deleted file mode 100644 index a85ac275..00000000 Binary files a/cmd/kk/pkg/plugins/network/hybridnet-0.6.6.tgz and /dev/null differ diff --git a/cmd/kk/pkg/plugins/network/modules.go b/cmd/kk/pkg/plugins/network/modules.go deleted file mode 100644 index db28cf6a..00000000 --- a/cmd/kk/pkg/plugins/network/modules.go +++ /dev/null @@ -1,386 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package network - -import ( - "path/filepath" - - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/network/templates" -) - -type DeployNetworkPluginModule struct { - common.KubeModule -} - -func (d *DeployNetworkPluginModule) Init() { - d.Name = "DeployNetworkPluginModule" - d.Desc = "Deploy cluster network plugin" - - switch d.KubeConf.Cluster.Network.Plugin { - case common.Calico: - d.Tasks = deployCalico(d) - case common.Flannel: - d.Tasks = deployFlannel(d) - case common.Cilium: - d.Tasks = deployCilium(d) - case common.Kubeovn: - d.Tasks = deployKubeOVN(d) - case common.Hybridnet: - d.Tasks = deployHybridnet(d) - default: - return - } - if d.KubeConf.Cluster.Network.EnableMultusCNI() { - d.Tasks = append(d.Tasks, deployMultus(d)...) - } -} - -func deployMultus(d *DeployNetworkPluginModule) []task.Interface { - generateMultus := &task.RemoteTask{ - Name: "GenerateMultus", - Desc: "Generate multus cni", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &OldK8sVersion{Not: true}, - }, - Action: &action.Template{ - Template: templates.Multus, - Dst: filepath.Join(common.KubeConfigDir, templates.Multus.Name()), - Data: util.Data{ - "MultusImage": images.GetImage(d.Runtime, d.KubeConf, "multus").ImageName(), - }, - }, - Parallel: true, - } - deploy := &task.RemoteTask{ - Name: "DeployMultus", - Desc: "Deploy multus", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(DeployNetworkMultusPlugin), - Parallel: true, - Retry: 5, - } - return []task.Interface{ - generateMultus, - deploy, - } -} - -func deployCalico(d *DeployNetworkPluginModule) []task.Interface { - generateCalicoOld := &task.RemoteTask{ - Name: "GenerateCalico", - Desc: "Generate calico", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(OldK8sVersion), - }, - Action: &action.Template{ - Template: templates.CalicoOld, - Dst: filepath.Join(common.KubeConfigDir, templates.CalicoOld.Name()), - Data: util.Data{ - "KubePodsCIDR": d.KubeConf.Cluster.Network.KubePodsCIDR, - "CalicoCniImage": images.GetImage(d.Runtime, d.KubeConf, "calico-cni").ImageName(), - "CalicoNodeImage": images.GetImage(d.Runtime, d.KubeConf, "calico-node").ImageName(), - "CalicoFlexvolImage": images.GetImage(d.Runtime, d.KubeConf, "calico-flexvol").ImageName(), - "CalicoControllersImage": images.GetImage(d.Runtime, d.KubeConf, "calico-kube-controllers").ImageName(), - "TyphaEnabled": len(d.Runtime.GetHostsByRole(common.K8s)) > 50 || d.KubeConf.Cluster.Network.Calico.Typha(), - "VethMTU": d.KubeConf.Cluster.Network.Calico.VethMTU, - "NodeCidrMaskSize": d.KubeConf.Cluster.Kubernetes.NodeCidrMaskSize, - "IPIPMode": d.KubeConf.Cluster.Network.Calico.IPIPMode, - "VXLANMode": d.KubeConf.Cluster.Network.Calico.VXLANMode, - }, - }, - Parallel: true, - } - - generateCalicoNew := &task.RemoteTask{ - Name: "GenerateCalico", - Desc: "Generate calico", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - &OldK8sVersion{Not: true}, - }, - Action: &action.Template{ - Template: templates.CalicoNew, - Dst: filepath.Join(common.KubeConfigDir, templates.CalicoNew.Name()), - Data: util.Data{ - "KubePodsCIDR": d.KubeConf.Cluster.Network.KubePodsCIDR, - "CalicoCniImage": images.GetImage(d.Runtime, d.KubeConf, "calico-cni").ImageName(), - "CalicoNodeImage": images.GetImage(d.Runtime, d.KubeConf, "calico-node").ImageName(), - "CalicoFlexvolImage": images.GetImage(d.Runtime, d.KubeConf, "calico-flexvol").ImageName(), - "CalicoControllersImage": images.GetImage(d.Runtime, d.KubeConf, "calico-kube-controllers").ImageName(), - "CalicoTyphaImage": images.GetImage(d.Runtime, d.KubeConf, "calico-typha").ImageName(), - "TyphaEnabled": len(d.Runtime.GetHostsByRole(common.K8s)) > 50 || d.KubeConf.Cluster.Network.Calico.Typha(), - "VethMTU": d.KubeConf.Cluster.Network.Calico.VethMTU, - "NodeCidrMaskSize": d.KubeConf.Cluster.Kubernetes.NodeCidrMaskSize, - "IPIPMode": d.KubeConf.Cluster.Network.Calico.IPIPMode, - "VXLANMode": d.KubeConf.Cluster.Network.Calico.VXLANMode, - "ConatinerManagerIsIsula": d.KubeConf.Cluster.Kubernetes.ContainerManager == "isula", - "IPV4POOLNATOUTGOING": d.KubeConf.Cluster.Network.Calico.EnableIPV4POOL_NAT_OUTGOING(), - "DefaultIPPOOL": d.KubeConf.Cluster.Network.Calico.EnableDefaultIPPOOL(), - }, - }, - Parallel: true, - } - - deploy := &task.RemoteTask{ - Name: "DeployCalico", - Desc: "Deploy calico", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(DeployNetworkPlugin), - Parallel: true, - Retry: 5, - } - - if K8sVersionAtLeast(d.KubeConf.Cluster.Kubernetes.Version, "v1.16.0") { - return []task.Interface{ - generateCalicoNew, - deploy, - } - } else { - return []task.Interface{ - generateCalicoOld, - deploy, - } - } -} - -func deployFlannel(d *DeployNetworkPluginModule) []task.Interface { - generateFlannelPSP := &task.RemoteTask{ - Name: "GenerateFlannel", - Desc: "Generate flannel", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: &action.Template{ - Template: templates.FlannelPSP, - Dst: filepath.Join(common.KubeConfigDir, templates.FlannelPSP.Name()), - Data: util.Data{ - "KubePodsCIDR": d.KubeConf.Cluster.Network.KubePodsCIDR, - "FlannelImage": images.GetImage(d.Runtime, d.KubeConf, "flannel").ImageName(), - "FlannelPluginImage": images.GetImage(d.Runtime, d.KubeConf, "flannel-cni-plugin").ImageName(), - "BackendMode": d.KubeConf.Cluster.Network.Flannel.BackendMode, - }, - }, - Parallel: true, - } - generateFlannelPS := &task.RemoteTask{ - Name: "GenerateFlannel", - Desc: "Generate flannel", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: &action.Template{ - Template: templates.FlannelPS, - Dst: filepath.Join(common.KubeConfigDir, templates.FlannelPS.Name()), - Data: util.Data{ - "KubePodsCIDR": d.KubeConf.Cluster.Network.KubePodsCIDR, - "FlannelImage": images.GetImage(d.Runtime, d.KubeConf, "flannel").ImageName(), - "FlannelPluginImage": images.GetImage(d.Runtime, d.KubeConf, "flannel-cni-plugin").ImageName(), - "BackendMode": d.KubeConf.Cluster.Network.Flannel.BackendMode, - }, - }, - Parallel: true, - } - - deploy := &task.RemoteTask{ - Name: "DeployFlannel", - Desc: "Deploy flannel", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(DeployNetworkPlugin), - Parallel: true, - Retry: 5, - } - - if K8sVersionAtLeast(d.KubeConf.Cluster.Kubernetes.Version, "v1.25.0") { - return []task.Interface{ - generateFlannelPS, - deploy, - } - } else { - return []task.Interface{ - generateFlannelPSP, - deploy, - } - } -} - -func deployCilium(d *DeployNetworkPluginModule) []task.Interface { - - releaseCiliumChart := &task.LocalTask{ - Name: "GenerateCiliumChart", - Desc: "Generate cilium chart", - Action: new(ReleaseCiliumChart), - } - - syncCiliumChart := &task.RemoteTask{ - Name: "SyncCiliumChart", - Desc: "Synchronize cilium chart", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(SyncCiliumChart), - Parallel: true, - Retry: 2, - } - - deploy := &task.RemoteTask{ - Name: "DeployCilium", - Desc: "Deploy cilium", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(DeployCilium), - Parallel: true, - Retry: 5, - } - - return []task.Interface{ - releaseCiliumChart, - syncCiliumChart, - deploy, - } -} - -func deployKubeOVN(d *DeployNetworkPluginModule) []task.Interface { - label := &task.RemoteTask{ - Name: "LabelNode", - Desc: "Label node", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(LabelNode), - Parallel: true, - } - - ssl := &task.RemoteTask{ - Name: "GenerateSSl", - Desc: "Generate ssl", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(EnableSSL), - }, - Action: new(GenerateSSL), - Parallel: true, - } - - generateKubeOVN := &task.RemoteTask{ - Name: "GenerateKubeOVN", - Desc: "Generate kube-ovn", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(GenerateKubeOVN), - Parallel: true, - } - - deploy := &task.RemoteTask{ - Name: "DeployKubeOVN", - Desc: "Deploy kube-ovn", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(DeployKubeovnPlugin), - Parallel: true, - Retry: 5, - } - - kubectlKo := &task.RemoteTask{ - Name: "GenerateKubectlKo", - Desc: "Generate kubectl-ko", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Action: &action.Template{ - Template: templates.KubectlKo, - Dst: filepath.Join(common.BinDir, templates.KubectlKo.Name()), - }, - Parallel: true, - } - - chmod := &task.RemoteTask{ - Name: "ChmodKubectlKo", - Desc: "Chmod kubectl-ko", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Action: new(ChmodKubectlKo), - Parallel: true, - } - - return []task.Interface{ - label, - ssl, - generateKubeOVN, - deploy, - kubectlKo, - chmod, - } -} - -func deployHybridnet(d *DeployNetworkPluginModule) []task.Interface { - - releaseHybridnetChart := &task.LocalTask{ - Name: "GenerateHybridnetChart", - Desc: "Generate hybridnet chart", - Action: new(ReleaseHybridnetChart), - } - - syncHybridnetChart := &task.RemoteTask{ - Name: "SyncHybridnetChart", - Desc: "Synchronize hybridnet chart", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(SyncHybridnetChart), - Parallel: true, - Retry: 2, - } - - deploy := &task.RemoteTask{ - Name: "DeployHybridnet", - Desc: "Deploy hybridnet", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(DeployHybridnet), - Parallel: true, - Retry: 5, - } - - return []task.Interface{ - releaseHybridnetChart, - syncHybridnetChart, - deploy, - } -} - -func K8sVersionAtLeast(version string, compare string) bool { - cmp, err := versionutil.MustParseSemantic(version).Compare(compare) - if err != nil { - logger.Log.Fatal("unknown kubernetes version") - } - // old version - if cmp == -1 { - return false - } - // new version - return true -} diff --git a/cmd/kk/pkg/plugins/network/prepares.go b/cmd/kk/pkg/plugins/network/prepares.go deleted file mode 100644 index eff35115..00000000 --- a/cmd/kk/pkg/plugins/network/prepares.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package network - -import ( - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type OldK8sVersion struct { - common.KubePrepare - Not bool -} - -func (o *OldK8sVersion) PreCheck(_ connector.Runtime) (bool, error) { - cmp, err := versionutil.MustParseSemantic(o.KubeConf.Cluster.Kubernetes.Version).Compare("v1.16.0") - if err != nil { - return false, err - } - // old version - if cmp == -1 { - return !o.Not, nil - } - // new version - return o.Not, nil -} - -type EnableSSL struct { - common.KubePrepare -} - -func (e *EnableSSL) PreCheck(_ connector.Runtime) (bool, error) { - if e.KubeConf.Cluster.Network.Kubeovn.EnableSSL { - return true, nil - } - return false, nil -} diff --git a/cmd/kk/pkg/plugins/network/tasks.go b/cmd/kk/pkg/plugins/network/tasks.go deleted file mode 100644 index cd3cf341..00000000 --- a/cmd/kk/pkg/plugins/network/tasks.go +++ /dev/null @@ -1,427 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package network - -import ( - "embed" - "fmt" - "github.com/pkg/errors" - "io" - "os" - "path/filepath" - "strings" - "time" - - "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/network/templates" -) - -//go:embed cilium-1.11.7.tgz hybridnet-0.6.6.tgz - -var f embed.FS - -type ReleaseCiliumChart struct { - common.KubeAction -} - -func (r *ReleaseCiliumChart) Execute(runtime connector.Runtime) error { - fs, err := os.Create(fmt.Sprintf("%s/cilium.tgz", runtime.GetWorkDir())) - if err != nil { - return err - } - chartFile, err := f.Open("cilium-1.11.7.tgz") - if err != nil { - return err - } - defer chartFile.Close() - - _, err = io.Copy(fs, chartFile) - if err != nil { - return err - } - - return nil -} - -type SyncCiliumChart struct { - common.KubeAction -} - -func (s *SyncCiliumChart) Execute(runtime connector.Runtime) error { - src := filepath.Join(runtime.GetWorkDir(), "cilium.tgz") - dst := filepath.Join(common.TmpDir, "cilium.tgz") - if err := runtime.GetRunner().Scp(src, dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync cilium chart failed")) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mv %s/cilium.tgz /etc/kubernetes", common.TmpDir), true); err != nil { - return errors.Wrap(errors.WithStack(err), "sync cilium chart failed") - } - return nil -} - -type DeployCilium struct { - common.KubeAction -} - -func (d *DeployCilium) Execute(runtime connector.Runtime) error { - ciliumImage := images.GetImage(runtime, d.KubeConf, "cilium").ImageName() - ciliumOperatorImage := images.GetImage(runtime, d.KubeConf, "cilium-operator-generic").ImageName() - - cmd := fmt.Sprintf("/usr/local/bin/helm upgrade --install cilium /etc/kubernetes/cilium.tgz --namespace kube-system "+ - "--set operator.image.override=%s "+ - "--set operator.replicas=1 "+ - "--set image.override=%s "+ - "--set ipam.operator.clusterPoolIPv4PodCIDR=%s", ciliumOperatorImage, ciliumImage, d.KubeConf.Cluster.Network.KubePodsCIDR) - - if d.KubeConf.Cluster.Kubernetes.DisableKubeProxy { - cmd = fmt.Sprintf("%s --set kubeProxyReplacement=strict --set k8sServiceHost=%s --set k8sServicePort=%d", cmd, d.KubeConf.Cluster.ControlPlaneEndpoint.Address, d.KubeConf.Cluster.ControlPlaneEndpoint.Port) - } - - if _, err := runtime.GetRunner().SudoCmd(cmd, true); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy cilium failed") - } - return nil -} - -type DeployNetworkPlugin struct { - common.KubeAction -} - -func (d *DeployNetworkPlugin) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl apply -f /etc/kubernetes/network-plugin.yaml --force", true); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy network plugin failed") - } - return nil -} - -type DeployKubeovnPlugin struct { - common.KubeAction -} - -func (d *DeployKubeovnPlugin) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl apply -f /etc/kubernetes/kube-ovn-crd.yaml --force", true); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy kube-ovn-crd.yaml failed") - } - if _, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl apply -f /etc/kubernetes/ovn.yaml --force", true); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy ovn.yaml failed") - } - if _, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl apply -f /etc/kubernetes/kube-ovn.yaml --force", true); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy kube-ovn.yaml failed") - } - return nil -} - -type DeployNetworkMultusPlugin struct { - common.KubeAction -} - -func (d *DeployNetworkMultusPlugin) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl apply -f /etc/kubernetes/multus-network-plugin.yaml --force", true); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy multus network plugin failed") - } - return nil -} - -type LabelNode struct { - common.KubeAction -} - -func (l *LabelNode) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("/usr/local/bin/kubectl label no -l%s kube-ovn/role=master --overwrite", - l.KubeConf.Cluster.Network.Kubeovn.Label), - true); err != nil { - return errors.Wrap(errors.WithStack(err), "label kubeovn/role=master in master node failed") - } - return nil -} - -type GenerateSSL struct { - common.KubeAction -} - -func (g *GenerateSSL) Execute(runtime connector.Runtime) error { - if exist, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl get secret -n kube-system kube-ovn-tls --ignore-not-found", - true); err != nil { - return errors.Wrap(errors.WithStack(err), "find ovn secret failed") - } else if exist != "" { - return nil - } - - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("docker run --rm -v %s:/etc/ovn %s bash generate-ssl.sh", - runtime.GetWorkDir(), images.GetImage(runtime, g.KubeConf, "kubeovn").ImageName()), - true); err != nil { - return errors.Wrap(errors.WithStack(err), "generate ovn secret failed") - } - - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("/usr/local/bin/kubectl create secret generic -n kube-system kube-ovn-tls "+ - "--from-file=cacert=%s/cacert.pem "+ - "--from-file=cert=%s/ovn-cert.pem "+ - "--from-file=key=%s/ovn-privkey.pem", - runtime.GetWorkDir(), runtime.GetWorkDir(), runtime.GetWorkDir()), - true); err != nil { - return errors.Wrap(errors.WithStack(err), "create ovn secret failed") - } - - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("rm -rf %s/cacert.pem %s/ovn-cert.pem %s/ovn-privkey.pem %s/ovn-req.pem", - runtime.GetWorkDir(), runtime.GetWorkDir(), runtime.GetWorkDir(), runtime.GetWorkDir()), - true); err != nil { - return errors.Wrap(errors.WithStack(err), "remove generated ovn secret file failed") - } - - return nil -} - -type GenerateKubeOVN struct { - common.KubeAction -} - -func (g *GenerateKubeOVN) Execute(runtime connector.Runtime) error { - address, err := runtime.GetRunner().Cmd( - "/usr/local/bin/kubectl get no -lkube-ovn/role=master --no-headers -o wide | awk '{print $6}' | tr \\\\n ','", - true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "get kube-ovn label node address failed") - } - - count, err := runtime.GetRunner().Cmd( - fmt.Sprintf("/usr/local/bin/kubectl get no -l%s --no-headers -o wide | wc -l | sed 's/ //g'", - g.KubeConf.Cluster.Network.Kubeovn.Label), true) - if err != nil { - return errors.Wrap(errors.WithStack(err), "count kube-ovn label nodes num failed") - } - - if count == "0" { - return fmt.Errorf("no node with label: %s", g.KubeConf.Cluster.Network.Kubeovn.Label) - } - - templateAction := action.Template{ - Template: templates.KubeOvnCrd, - Dst: filepath.Join(common.KubeConfigDir, templates.KubeOvnCrd.Name()), - } - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - - templateAction = action.Template{ - Template: templates.OVN, - Dst: filepath.Join(common.KubeConfigDir, templates.OVN.Name()), - Data: util.Data{ - "Address": address, - "Count": count, - "KubeovnImage": images.GetImage(runtime, g.KubeConf, "kubeovn").ImageName(), - "TunnelType": g.KubeConf.Cluster.Network.Kubeovn.TunnelType, - "DpdkMode": g.KubeConf.Cluster.Network.Kubeovn.Dpdk.DpdkMode, - "DpdkVersion": g.KubeConf.Cluster.Network.Kubeovn.Dpdk.DpdkVersion, - "OvnVersion": v1alpha2.DefaultKubeovnVersion, - "EnableSSL": g.KubeConf.Cluster.Network.Kubeovn.EnableSSL, - "HwOffload": g.KubeConf.Cluster.Network.Kubeovn.OvsOvn.HwOffload, - "SvcYamlIpfamilypolicy": g.KubeConf.Cluster.Network.Kubeovn.SvcYamlIpfamilypolicy, - }, - } - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - - templateAction = action.Template{ - Template: templates.KubeOvn, - Dst: filepath.Join(common.KubeConfigDir, templates.KubeOvn.Name()), - Data: util.Data{ - "Address": address, - "Count": count, - "KubeovnImage": images.GetImage(runtime, g.KubeConf, "kubeovn").ImageName(), - "PodCIDR": g.KubeConf.Cluster.Network.KubePodsCIDR, - "SvcCIDR": g.KubeConf.Cluster.Network.KubeServiceCIDR, - "JoinCIDR": g.KubeConf.Cluster.Network.Kubeovn.JoinCIDR, - "PodGateway": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnController.PodGateway, - "CheckGateway": g.KubeConf.Cluster.Network.Kubeovn.KubeovnCheckGateway(), - "LogicalGateway": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnController.LogicalGateway, - "PingExternalAddress": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnPinger.PingerExternalAddress, - "PingExternalDNS": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnPinger.PingerExternalDomain, - "NetworkType": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnController.NetworkType, - "TunnelType": g.KubeConf.Cluster.Network.Kubeovn.TunnelType, - "ExcludeIps": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnController.ExcludeIps, - "PodNicType": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnController.PodNicType, - "VlanID": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnController.VlanID, - "VlanInterfaceName": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnController.VlanInterfaceName, - "Iface": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnCni.Iface, - "EnableSSL": g.KubeConf.Cluster.Network.Kubeovn.EnableSSL, - "EnableMirror": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnCni.EnableMirror, - "EnableLB": g.KubeConf.Cluster.Network.Kubeovn.KubeovnEnableLB(), - "EnableNP": g.KubeConf.Cluster.Network.Kubeovn.KubeovnEnableNP(), - "EnableEipSnat": g.KubeConf.Cluster.Network.Kubeovn.KubeovnEnableEipSnat(), - "EnableExternalVPC": g.KubeConf.Cluster.Network.Kubeovn.KubeovnEnableExternalVPC(), - "SvcYamlIpfamilypolicy": g.KubeConf.Cluster.Network.Kubeovn.SvcYamlIpfamilypolicy, - "DpdkTunnelIface": g.KubeConf.Cluster.Network.Kubeovn.Dpdk.DpdkTunnelIface, - "CNIConfigPriority": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnCni.CNIConfigPriority, - "Modules": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnCni.Modules, - "RPMs": g.KubeConf.Cluster.Network.Kubeovn.KubeOvnCni.RPMs, - }, - } - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - - return nil -} - -type ChmodKubectlKo struct { - common.KubeAction -} - -func (c *ChmodKubectlKo) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd( - fmt.Sprintf("chmod +x %s", filepath.Join(common.BinDir, templates.KubectlKo.Name())), false); err != nil { - return errors.Wrap(errors.WithStack(err), "chmod +x kubectl-ko failed") - } - return nil -} - -// ReleaseHybridnetChart is used to release hybridnet chart to local path -type ReleaseHybridnetChart struct { - common.KubeAction -} - -func (r *ReleaseHybridnetChart) Execute(runtime connector.Runtime) error { - fs, err := os.Create(fmt.Sprintf("%s/hybridnet.tgz", runtime.GetWorkDir())) - if err != nil { - return err - } - chartFile, err := f.Open("hybridnet-0.6.6.tgz") - if err != nil { - return err - } - defer chartFile.Close() - - _, err = io.Copy(fs, chartFile) - if err != nil { - return err - } - - return nil -} - -// SyncHybridnetChart is used to sync hybridnet chart to contronplane -type SyncHybridnetChart struct { - common.KubeAction -} - -func (s *SyncHybridnetChart) Execute(runtime connector.Runtime) error { - src := filepath.Join(runtime.GetWorkDir(), "hybridnet.tgz") - dst := filepath.Join(common.TmpDir, "hybridnet.tgz") - if err := runtime.GetRunner().Scp(src, dst); err != nil { - return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync hybridnet chart failed")) - } - if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mv %s/hybridnet.tgz /etc/kubernetes", common.TmpDir), true); err != nil { - return errors.Wrap(errors.WithStack(err), "sync hybrident chart failed") - } - return nil -} - -type DeployHybridnet struct { - common.KubeAction -} - -func (d *DeployHybridnet) Execute(runtime connector.Runtime) error { - - cmd := fmt.Sprintf("/usr/local/bin/helm upgrade --install hybridnet /etc/kubernetes/hybridnet.tgz --namespace kube-system "+ - "--set images.hybridnet.image=%s/%s "+ - "--set images.hybridnet.tag=%s "+ - "--set images.registryURL=%s ", - images.GetImage(runtime, d.KubeConf, "hybridnet").ImageNamespace(), - images.GetImage(runtime, d.KubeConf, "hybridnet").Repo, - images.GetImage(runtime, d.KubeConf, "hybridnet").Tag, - images.GetImage(runtime, d.KubeConf, "hybridnet").ImageRegistryAddr(), - ) - - if d.KubeConf.Cluster.Network.Hybridnet.EnableInit() { - cmd = fmt.Sprintf("%s --set init.cidr=%s", cmd, d.KubeConf.Cluster.Network.KubePodsCIDR) - } else { - cmd = fmt.Sprintf("%s --set init=null", cmd) - } - - if !d.KubeConf.Cluster.Network.Hybridnet.NetworkPolicy() { - cmd = fmt.Sprintf("%s --set daemon.enableNetworkPolicy=false", cmd) - } - - if d.KubeConf.Cluster.Network.Hybridnet.DefaultNetworkType != "" { - cmd = fmt.Sprintf("%s --set defaultNetworkType=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.DefaultNetworkType) - } - - if d.KubeConf.Cluster.Network.Hybridnet.PreferBGPInterfaces != "" { - cmd = fmt.Sprintf("%s --set daemon.preferBGPInterfaces=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.PreferBGPInterfaces) - } - - if d.KubeConf.Cluster.Network.Hybridnet.PreferVlanInterfaces != "" { - cmd = fmt.Sprintf("%s --set daemon.preferVlanInterfaces=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.PreferVlanInterfaces) - } - - if d.KubeConf.Cluster.Network.Hybridnet.PreferVxlanInterfaces != "" { - cmd = fmt.Sprintf("%s --set daemon.preferVxlanInterfaces=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.PreferVxlanInterfaces) - } - - if _, err := runtime.GetRunner().SudoCmd(cmd, true); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy hybridnet failed") - } - - if len(d.KubeConf.Cluster.Network.Hybridnet.Networks) != 0 { - templateAction := action.Template{ - Template: templates.HybridnetNetworks, - Dst: filepath.Join(common.KubeConfigDir, templates.HybridnetNetworks.Name()), - Data: util.Data{ - "Networks": d.KubeConf.Cluster.Network.Hybridnet.Networks, - }, - } - - templateAction.Init(nil, nil) - if err := templateAction.Execute(runtime); err != nil { - return err - } - - for i := 0; i < 30; i++ { - fmt.Println("Waiting for hybridnet webhook running ... ", i+1) - time.Sleep(10 * time.Second) - output, _ := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl get pod -n kube-system -l app=hybridnet,component=webhook | grep Running", false) - if strings.Contains(output, "1/1") { - time.Sleep(50 * time.Second) - break - } - } - - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/hybridnet-networks.yaml", true); err != nil { - return errors.Wrap(errors.WithStack(err), "apply hybridnet networks failed") - } - } - return nil -} diff --git a/cmd/kk/pkg/plugins/network/templates/calico_v1.16+.go b/cmd/kk/pkg/plugins/network/templates/calico_v1.16+.go deleted file mode 100644 index d803b611..00000000 --- a/cmd/kk/pkg/plugins/network/templates/calico_v1.16+.go +++ /dev/null @@ -1,5207 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var CalicoNew = template.Must(template.New("network-plugin.yaml").Parse( - dedent.Dedent(` ---- -# Source: calico/templates/calico-kube-controllers.yaml -# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict - -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers -spec: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers - -{{ if .TyphaEnabled }} ---- -# Source: calico/templates/calico-typha.yaml -# This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict - -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha -spec: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: calico-typha - -{{ end }} ---- -# Source: calico/templates/calico-kube-controllers.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-kube-controllers - namespace: kube-system ---- -# Source: calico/templates/calico-node.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system ---- -# Source: calico/templates/calico-node.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-cni-plugin - namespace: kube-system ---- -# Source: calico/templates/calico-config.yaml -# This ConfigMap is used to configure a self-hosted Calico installation. -kind: ConfigMap -apiVersion: v1 -metadata: - name: calico-config - namespace: kube-system -data: - # You must set a non-zero value for Typha replicas below. - typha_service_name: {{ if .TyphaEnabled }}"calico-typha"{{ else }}"none"{{ end }} - # Configure the backend to use. - calico_backend: "bird" - - # Configure the MTU to use for workload interfaces and tunnels. - # By default, MTU is auto-detected, and explicitly setting this field should not be required. - # You can override auto-detection by providing a non-zero value. - veth_mtu: "{{ .VethMTU }}" - - # The CNI network configuration to install on each node. The special - # values in this config will be automatically populated. - cni_network_config: |- - { - "name": "k8s-pod-network", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "calico", - "log_level": "info", - "log_file_path": "/var/log/calico/cni/cni.log", - "datastore_type": "kubernetes", - "nodename": "__KUBERNETES_NODE_NAME__", - "mtu": __CNI_MTU__, - "ipam": { - "type": "calico-ipam" - }, - "policy": { - "type": "k8s" - }, - "kubernetes": { - "kubeconfig": "__KUBECONFIG_FILEPATH__" - } - }, - { - "type": "portmap", - "snat": true, - "capabilities": {"portMappings": true} - }, - { - "type": "bandwidth", - "capabilities": {"bandwidth": true} - } - ] - } ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: bgpconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BGPConfiguration - listKind: BGPConfigurationList - plural: bgpconfigurations - singular: bgpconfiguration - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: BGPConfiguration contains the configuration for any BGP routing. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPConfigurationSpec contains the values of the BGP configuration. - properties: - asNumber: - description: 'ASNumber is the default AS number used by a node. [Default: - 64512]' - format: int32 - type: integer - bindMode: - description: BindMode indicates whether to listen for BGP connections - on all addresses (None) or only on the node's canonical IP address - Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen - for BGP connections on all addresses. - type: string - communities: - description: Communities is a list of BGP community values and their - arbitrary names for tagging routes. - items: - description: Community contains standard or large community value - and its name. - properties: - name: - description: Name given to community value. - type: string - value: - description: Value must be of format aa:nn or aa:nn:mm. - For standard community use aa:nn format, where aa and - nn are 16 bit number. For large community use aa:nn:mm - format, where aa, nn and mm are 32 bit number. Where, - aa is an AS Number, nn and mm are per-AS identifier. - pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ - type: string - type: object - type: array - ignoredInterfaces: - description: IgnoredInterfaces indicates the network interfaces that - needs to be excluded when reading device routes. - items: - type: string - type: array - listenPort: - description: ListenPort is the port where BGP protocol should listen. - Defaults to 179 - maximum: 65535 - minimum: 1 - type: integer - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: INFO]' - type: string - nodeMeshMaxRestartTime: - description: Time to allow for software restart for node-to-mesh peerings. When - specified, this is configured as the graceful restart timeout. When - not specified, the BIRD default of 120s is used. This field can - only be set on the default BGPConfiguration instance and requires - that NodeMesh is enabled - type: string - nodeMeshPassword: - description: Optional BGP password for full node-to-mesh peerings. - This field can only be set on the default BGPConfiguration instance - and requires that NodeMesh is enabled - properties: - secretKeyRef: - description: Selects a key of a secret in the node pod's namespace. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must be - defined - type: boolean - required: - - key - type: object - type: object - nodeToNodeMeshEnabled: - description: 'NodeToNodeMeshEnabled sets whether full node to node - BGP mesh is enabled. [Default: true]' - type: boolean - prefixAdvertisements: - description: PrefixAdvertisements contains per-prefix advertisement - configuration. - items: - description: PrefixAdvertisement configures advertisement properties - for the specified CIDR. - properties: - cidr: - description: CIDR for which properties should be advertised. - type: string - communities: - description: Communities can be list of either community names - already defined in Specs.Communities or community value - of format aa:nn or aa:nn:mm. For standard community use - aa:nn format, where aa and nn are 16 bit number. For - large community use aa:nn:mm format, where aa, nn and - mm are 32 bit number. Where,aa is an AS Number, nn and - mm are per-AS identifier. - items: - type: string - type: array - type: object - type: array - serviceClusterIPs: - description: ServiceClusterIPs are the CIDR blocks from which service - cluster IPs are allocated. If specified, Calico will advertise these - blocks, as well as any cluster IPs within them. - items: - description: ServiceClusterIPBlock represents a single allowed ClusterIP - CIDR block. - properties: - cidr: - type: string - type: object - type: array - serviceExternalIPs: - description: ServiceExternalIPs are the CIDR blocks for Kubernetes - Service External IPs. Kubernetes Service ExternalIPs will only be - advertised if they are within one of these blocks. - items: - description: ServiceExternalIPBlock represents a single allowed - External IP CIDR block. - properties: - cidr: - type: string - type: object - type: array - serviceLoadBalancerIPs: - description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes - Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress - IPs will only be advertised if they are within one of these blocks. - items: - description: ServiceLoadBalancerIPBlock represents a single allowed - LoadBalancer IP CIDR block. - properties: - cidr: - type: string - type: object - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: bgpfilters.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BGPFilter - listKind: BGPFilterList - plural: bgpfilters - singular: bgpfilter - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPFilterSpec contains the IPv4 and IPv6 filter rules of - the BGP Filter. - properties: - exportV4: - description: The ordered set of IPv4 BGPFilter rules acting on exporting - routes to a peer. - items: - description: BGPFilterRuleV4 defines a BGP filter rule consisting - a single IPv4 CIDR block and a filter action for this CIDR. - properties: - action: - type: string - cidr: - type: string - matchOperator: - type: string - required: - - action - - cidr - - matchOperator - type: object - type: array - exportV6: - description: The ordered set of IPv6 BGPFilter rules acting on exporting - routes to a peer. - items: - description: BGPFilterRuleV6 defines a BGP filter rule consisting - a single IPv6 CIDR block and a filter action for this CIDR. - properties: - action: - type: string - cidr: - type: string - matchOperator: - type: string - required: - - action - - cidr - - matchOperator - type: object - type: array - importV4: - description: The ordered set of IPv4 BGPFilter rules acting on importing - routes from a peer. - items: - description: BGPFilterRuleV4 defines a BGP filter rule consisting - a single IPv4 CIDR block and a filter action for this CIDR. - properties: - action: - type: string - cidr: - type: string - matchOperator: - type: string - required: - - action - - cidr - - matchOperator - type: object - type: array - importV6: - description: The ordered set of IPv6 BGPFilter rules acting on importing - routes from a peer. - items: - description: BGPFilterRuleV6 defines a BGP filter rule consisting - a single IPv6 CIDR block and a filter action for this CIDR. - properties: - action: - type: string - cidr: - type: string - matchOperator: - type: string - required: - - action - - cidr - - matchOperator - type: object - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: bgppeers.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BGPPeer - listKind: BGPPeerList - plural: bgppeers - singular: bgppeer - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPPeerSpec contains the specification for a BGPPeer resource. - properties: - asNumber: - description: The AS Number of the peer. - format: int32 - type: integer - filters: - description: The ordered set of BGPFilters applied on this BGP peer. - items: - type: string - type: array - keepOriginalNextHop: - description: Option to keep the original nexthop field when routes - are sent to a BGP Peer. Setting "true" configures the selected BGP - Peers node to use the "next hop keep;" instead of "next hop self;"(default) - in the specific branch of the Node on "bird.cfg". - type: boolean - maxRestartTime: - description: Time to allow for software restart. When specified, - this is configured as the graceful restart timeout. When not specified, - the BIRD default of 120s is used. - type: string - node: - description: The node name identifying the Calico node instance that - is targeted by this peer. If this is not set, and no nodeSelector - is specified, then this BGP peer selects all nodes in the cluster. - type: string - nodeSelector: - description: Selector for the nodes that should have this peering. When - this is set, the Node field must be empty. - type: string - numAllowedLocalASNumbers: - description: Maximum number of local AS numbers that are allowed in - the AS path for received routes. This removes BGP loop prevention - and should only be used if absolutely necesssary. - format: int32 - type: integer - password: - description: Optional BGP password for the peerings generated by this - BGPPeer resource. - properties: - secretKeyRef: - description: Selects a key of a secret in the node pod's namespace. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must be - defined - type: boolean - required: - - key - type: object - type: object - peerIP: - description: The IP address of the peer followed by an optional port - number to peer with. If port number is given, format should be []:port - or : for IPv4. If optional port number is not set, - and this peer IP and ASNumber belongs to a calico/node with ListenPort - set in BGPConfiguration, then we use that port to peer. - type: string - peerSelector: - description: Selector for the remote nodes to peer with. When this - is set, the PeerIP and ASNumber fields must be empty. For each - peering between the local node and selected remote nodes, we configure - an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, - and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The - remote AS number comes from the remote node's NodeBGPSpec.ASNumber, - or the global default if that is not set. - type: string - reachableBy: - description: Add an exact, i.e. /32, static route toward peer IP in - order to prevent route flapping. ReachableBy contains the address - of the gateway which peer can be reached by. - type: string - sourceAddress: - description: Specifies whether and how to configure a source address - for the peerings generated by this BGPPeer resource. Default value - "UseNodeIP" means to configure the node IP as the source address. "None" - means not to configure a source address. - type: string - ttlSecurity: - description: TTLSecurity enables the generalized TTL security mechanism - (GTSM) which protects against spoofed packets by ignoring received - packets with a smaller than expected TTL value. The provided value - is the number of hops (edges) between the peers. - type: integer - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: blockaffinities.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BlockAffinity - listKind: BlockAffinityList - plural: blockaffinities - singular: blockaffinity - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BlockAffinitySpec contains the specification for a BlockAffinity - resource. - properties: - cidr: - type: string - deleted: - description: Deleted indicates that this block affinity is being deleted. - This field is a string for compatibility with older releases that - mistakenly treat this field as a string. - type: string - node: - type: string - state: - type: string - required: - - cidr - - deleted - - node - - state - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: caliconodestatuses.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: CalicoNodeStatus - listKind: CalicoNodeStatusList - plural: caliconodestatuses - singular: caliconodestatus - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: CalicoNodeStatusSpec contains the specification for a CalicoNodeStatus - resource. - properties: - classes: - description: Classes declares the types of information to monitor - for this calico/node, and allows for selective status reporting - about certain subsets of information. - items: - type: string - type: array - node: - description: The node name identifies the Calico node instance for - node status. - type: string - updatePeriodSeconds: - description: UpdatePeriodSeconds is the period at which CalicoNodeStatus - should be updated. Set to 0 to disable CalicoNodeStatus refresh. - Maximum update period is one day. - format: int32 - type: integer - type: object - status: - description: CalicoNodeStatusStatus defines the observed state of CalicoNodeStatus. - No validation needed for status since it is updated by Calico. - properties: - agent: - description: Agent holds agent status on the node. - properties: - birdV4: - description: BIRDV4 represents the latest observed status of bird4. - properties: - lastBootTime: - description: LastBootTime holds the value of lastBootTime - from bird.ctl output. - type: string - lastReconfigurationTime: - description: LastReconfigurationTime holds the value of lastReconfigTime - from bird.ctl output. - type: string - routerID: - description: Router ID used by bird. - type: string - state: - description: The state of the BGP Daemon. - type: string - version: - description: Version of the BGP daemon - type: string - type: object - birdV6: - description: BIRDV6 represents the latest observed status of bird6. - properties: - lastBootTime: - description: LastBootTime holds the value of lastBootTime - from bird.ctl output. - type: string - lastReconfigurationTime: - description: LastReconfigurationTime holds the value of lastReconfigTime - from bird.ctl output. - type: string - routerID: - description: Router ID used by bird. - type: string - state: - description: The state of the BGP Daemon. - type: string - version: - description: Version of the BGP daemon - type: string - type: object - type: object - bgp: - description: BGP holds node BGP status. - properties: - numberEstablishedV4: - description: The total number of IPv4 established bgp sessions. - type: integer - numberEstablishedV6: - description: The total number of IPv6 established bgp sessions. - type: integer - numberNotEstablishedV4: - description: The total number of IPv4 non-established bgp sessions. - type: integer - numberNotEstablishedV6: - description: The total number of IPv6 non-established bgp sessions. - type: integer - peersV4: - description: PeersV4 represents IPv4 BGP peers status on the node. - items: - description: CalicoNodePeer contains the status of BGP peers - on the node. - properties: - peerIP: - description: IP address of the peer whose condition we are - reporting. - type: string - since: - description: Since the state or reason last changed. - type: string - state: - description: State is the BGP session state. - type: string - type: - description: Type indicates whether this peer is configured - via the node-to-node mesh, or via en explicit global or - per-node BGPPeer object. - type: string - type: object - type: array - peersV6: - description: PeersV6 represents IPv6 BGP peers status on the node. - items: - description: CalicoNodePeer contains the status of BGP peers - on the node. - properties: - peerIP: - description: IP address of the peer whose condition we are - reporting. - type: string - since: - description: Since the state or reason last changed. - type: string - state: - description: State is the BGP session state. - type: string - type: - description: Type indicates whether this peer is configured - via the node-to-node mesh, or via en explicit global or - per-node BGPPeer object. - type: string - type: object - type: array - required: - - numberEstablishedV4 - - numberEstablishedV6 - - numberNotEstablishedV4 - - numberNotEstablishedV6 - type: object - lastUpdated: - description: LastUpdated is a timestamp representing the server time - when CalicoNodeStatus object last updated. It is represented in - RFC3339 form and is in UTC. - format: date-time - nullable: true - type: string - routes: - description: Routes reports routes known to the Calico BGP daemon - on the node. - properties: - routesV4: - description: RoutesV4 represents IPv4 routes on the node. - items: - description: CalicoNodeRoute contains the status of BGP routes - on the node. - properties: - destination: - description: Destination of the route. - type: string - gateway: - description: Gateway for the destination. - type: string - interface: - description: Interface for the destination - type: string - learnedFrom: - description: LearnedFrom contains information regarding - where this route originated. - properties: - peerIP: - description: If sourceType is NodeMesh or BGPPeer, IP - address of the router that sent us this route. - type: string - sourceType: - description: Type of the source where a route is learned - from. - type: string - type: object - type: - description: Type indicates if the route is being used for - forwarding or not. - type: string - type: object - type: array - routesV6: - description: RoutesV6 represents IPv6 routes on the node. - items: - description: CalicoNodeRoute contains the status of BGP routes - on the node. - properties: - destination: - description: Destination of the route. - type: string - gateway: - description: Gateway for the destination. - type: string - interface: - description: Interface for the destination - type: string - learnedFrom: - description: LearnedFrom contains information regarding - where this route originated. - properties: - peerIP: - description: If sourceType is NodeMesh or BGPPeer, IP - address of the router that sent us this route. - type: string - sourceType: - description: Type of the source where a route is learned - from. - type: string - type: object - type: - description: Type indicates if the route is being used for - forwarding or not. - type: string - type: object - type: array - type: object - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: clusterinformations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: ClusterInformation - listKind: ClusterInformationList - plural: clusterinformations - singular: clusterinformation - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: ClusterInformation contains the cluster specific information. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ClusterInformationSpec contains the values of describing - the cluster. - properties: - calicoVersion: - description: CalicoVersion is the version of Calico that the cluster - is running - type: string - clusterGUID: - description: ClusterGUID is the GUID of the cluster - type: string - clusterType: - description: ClusterType describes the type of the cluster - type: string - datastoreReady: - description: DatastoreReady is used during significant datastore migrations - to signal to components such as Felix that it should wait before - accessing the datastore. - type: boolean - variant: - description: Variant declares which variant of Calico should be active. - type: string - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: felixconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: FelixConfiguration - listKind: FelixConfigurationList - plural: felixconfigurations - singular: felixconfiguration - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: Felix Configuration contains the configuration for Felix. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: FelixConfigurationSpec contains the values of the Felix configuration. - properties: - allowIPIPPacketsFromWorkloads: - description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads - [Default: false]' - type: boolean - allowVXLANPacketsFromWorkloads: - description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads - [Default: false]' - type: boolean - awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enable" or "Disable". [Default: - DoNothing]' - enum: - - DoNothing - - Enable - - Disable - type: string - bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. [Default: - true]' - type: boolean - bpfDSROptoutCIDRs: - description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded - from DSR. That is, clients in those CIDRs will accesses nodeports - as if BPFExternalServiceMode was set to Tunnel. - items: - type: string - type: array - bpfDataIfacePattern: - description: BPFDataIfacePattern is a regular expression that controls - which interfaces Felix should attach BPF programs to in order to - catch traffic to/from the network. This needs to match the interfaces - that Calico workload traffic flows over as well as any interfaces - that handle incoming traffic to nodeports and services from outside - the cluster. It should not match the workload interfaces (usually - named cali...). - type: string - bpfDisableUnprivileged: - description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled - sysctl to disable unprivileged use of BPF. This ensures that unprivileged - users cannot access Calico''s BPF maps and cannot insert their own - BPF programs to interfere with Calico''s. [Default: true]' - type: boolean - bpfEnabled: - description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. - [Default: false]' - type: boolean - bpfEnforceRPF: - description: 'BPFEnforceRPF enforce strict RPF on all host interfaces - with BPF programs regardless of what is the per-interfaces or global - setting. Possible values are Disabled, Strict or Loose. [Default: - Loose]' - type: string - bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit - mark that is set on connections from an external client to a local - service. This mark allows us to control how packets of that connection - are routed within the host and how is routing interpreted by RPF - check. [Default: 0]' - type: integer - bpfExternalServiceMode: - description: 'BPFExternalServiceMode in BPF mode, controls how connections - from outside the cluster to services (node ports and cluster IPs) - are forwarded to remote workloads. If set to "Tunnel" then both - request and response traffic is tunneled to the remote node. If - set to "DSR", the request traffic is tunneled but the response traffic - is sent directly from the remote node. In "DSR" mode, the remote - node appears to use the IP of the ingress node; this requires a - permissive L2 network. [Default: Tunnel]' - type: string - bpfHostConntrackBypass: - description: 'BPFHostConntrackBypass Controls whether to bypass Linux - conntrack in BPF mode for workloads and services. [Default: true - - bypass Linux conntrack]' - type: boolean - bpfKubeProxyEndpointSlicesEnabled: - description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls - whether Felix's embedded kube-proxy accepts EndpointSlices or not. - type: boolean - bpfKubeProxyIptablesCleanupEnabled: - description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF - mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s - iptables chains. Should only be enabled if kube-proxy is not running. [Default: - true]' - type: boolean - bpfKubeProxyMinSyncPeriod: - description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the - minimum time between updates to the dataplane for Felix''s embedded - kube-proxy. Lower values give reduced set-up latency. Higher values - reduce Felix CPU usage by batching up more work. [Default: 1s]' - type: string - bpfL3IfacePattern: - description: BPFL3IfacePattern is a regular expression that allows - to list tunnel devices like wireguard or vxlan (i.e., L3 devices) - in addition to BPFDataIfacePattern. That is, tunnel interfaces not - created by Calico, that Calico workload traffic flows over as well - as any interfaces that handle incoming traffic to nodeports and - services from outside the cluster. - type: string - bpfLogLevel: - description: 'BPFLogLevel controls the log level of the BPF programs - when in BPF dataplane mode. One of "Off", "Info", or "Debug". The - logs are emitted to the BPF trace pipe, accessible with the command - tc exec bpf debug. [Default: Off].' - type: string - bpfMapSizeConntrack: - description: 'BPFMapSizeConntrack sets the size for the conntrack - map. This map must be large enough to hold an entry for each active - connection. Warning: changing the size of the conntrack map can - cause disruption.' - type: integer - bpfMapSizeIPSets: - description: BPFMapSizeIPSets sets the size for ipsets map. The IP - sets map must be large enough to hold an entry for each endpoint - matched by every selector in the source/destination matches in network - policy. Selectors such as "all()" can result in large numbers of - entries (one entry per endpoint in that case). - type: integer - bpfMapSizeIfState: - description: BPFMapSizeIfState sets the size for ifstate map. The - ifstate map must be large enough to hold an entry for each device - (host + workloads) on a host. - type: integer - bpfMapSizeNATAffinity: - type: integer - bpfMapSizeNATBackend: - description: BPFMapSizeNATBackend sets the size for nat back end map. - This is the total number of endpoints. This is mostly more than - the size of the number of services. - type: integer - bpfMapSizeNATFrontend: - description: BPFMapSizeNATFrontend sets the size for nat front end - map. FrontendMap should be large enough to hold an entry for each - nodeport, external IP and each port in each service. - type: integer - bpfMapSizeRoute: - description: BPFMapSizeRoute sets the size for the routes map. The - routes map should be large enough to hold one entry per workload - and a handful of entries per host (enough to cover its own IPs and - tunnel IPs). - type: integer - bpfPSNATPorts: - anyOf: - - type: integer - - type: string - description: 'BPFPSNATPorts sets the range from which we randomly - pick a port if there is a source port collision. This should be - within the ephemeral range as defined by RFC 6056 (1024–65535) and - preferably outside the ephemeral ranges used by common operating - systems. Linux uses 32768–60999, while others mostly use the IANA - defined range 49152–65535. It is not necessarily a problem if this - range overlaps with the operating systems. Both ends of the range - are inclusive. [Default: 20000:29999]' - pattern: ^.* - x-kubernetes-int-or-string: true - bpfPolicyDebugEnabled: - description: BPFPolicyDebugEnabled when true, Felix records detailed - information about the BPF policy programs, which can be examined - with the calico-bpf command-line tool. - type: boolean - chainInsertMode: - description: 'ChainInsertMode controls whether Felix hooks the kernel''s - top-level iptables chains by inserting a rule at the top of the - chain or by appending a rule at the bottom. insert is the safe default - since it prevents Calico''s rules from being bypassed. If you switch - to append mode, be sure that the other rules in the chains signal - acceptance by falling through to the Calico rules, otherwise the - Calico policy will be bypassed. [Default: insert]' - type: string - dataplaneDriver: - description: DataplaneDriver filename of the external dataplane driver - to use. Only used if UseInternalDataplaneDriver is set to false. - type: string - dataplaneWatchdogTimeout: - description: "DataplaneWatchdogTimeout is the readiness/liveness timeout - used for Felix's (internal) dataplane driver. Increase this value - if you experience spurious non-ready or non-live events when Felix - is under heavy load. Decrease the value to get felix to report non-live - or non-ready more quickly. [Default: 90s] \n Deprecated: replaced - by the generic HealthTimeoutOverrides." - type: string - debugDisableLogDropping: - type: boolean - debugMemoryProfilePath: - type: string - debugSimulateCalcGraphHangAfter: - type: string - debugSimulateDataplaneHangAfter: - type: string - defaultEndpointToHostAction: - description: 'DefaultEndpointToHostAction controls what happens to - traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables "DROP" action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables "INPUT" chain; - Calico will insert its rules at the top of that chain, then "RETURN" - packets to the "INPUT" chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' - type: string - deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. - type: integer - deviceRouteSourceAddress: - description: This is the IPv4 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. - type: string - deviceRouteSourceAddressIPv6: - description: This is the IPv6 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. - type: string - disableConntrackInvalidCheck: - type: boolean - endpointReportingDelay: - type: string - endpointReportingEnabled: - type: boolean - externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. - items: - type: string - type: array - failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports - and CIDRs that Felix will allow incoming traffic to host endpoints - on irrespective of the security policy. This is useful to avoid - accidentally cutting off a host with incorrect configuration. For - back-compatibility, if the protocol is not specified, it defaults - to "tcp". If a CIDR is not specified, it will allow traffic from - all addresses. To disable all inbound host ports, use the value - none. The default value allows ssh access and DHCP. [Default: tcp:22, - udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' - items: - description: ProtoPort is combination of protocol, port, and CIDR. - Protocol and port must be specified. - properties: - net: - type: string - port: - type: integer - protocol: - type: string - required: - - port - - protocol - type: object - type: array - failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports - and CIDRs that Felix will allow outgoing traffic from host endpoints - to irrespective of the security policy. This is useful to avoid - accidentally cutting off a host with incorrect configuration. For - back-compatibility, if the protocol is not specified, it defaults - to "tcp". If a CIDR is not specified, it will allow traffic from - all addresses. To disable all outbound host ports, use the value - none. The default value opens etcd''s standard ports to ensure that - Felix does not get cut off from etcd as well as allowing DHCP and - DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, - tcp:6667, udp:53, udp:67]' - items: - description: ProtoPort is combination of protocol, port, and CIDR. - Protocol and port must be specified. - properties: - net: - type: string - port: - type: integer - protocol: - type: string - required: - - port - - protocol - type: object - type: array - featureDetectOverride: - description: FeatureDetectOverride is used to override feature detection - based on auto-detected platform capabilities. Values are specified - in a comma separated list with no spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". "true" - or "false" will force the feature, empty or omitted values are auto-detected. - type: string - featureGates: - description: FeatureGates is used to enable or disable tech-preview - Calico features. Values are specified in a comma separated list - with no spaces, example; "BPFConnectTimeLoadBalancingWorkaround=enabled,XyZ=false". - This is used to enable features that are not fully production ready. - type: string - floatingIPs: - description: FloatingIPs configures whether or not Felix will program - non-OpenStack floating IP addresses. (OpenStack-derived floating - IPs are always programmed, regardless of this setting.) - enum: - - Enabled - - Disabled - type: string - genericXDPEnabled: - description: 'GenericXDPEnabled enables Generic XDP so network cards - that don''t support XDP offload or driver modes can use XDP. This - is not recommended since it doesn''t provide better performance - than iptables. [Default: false]' - type: boolean - healthEnabled: - type: boolean - healthHost: - type: string - healthPort: - type: integer - healthTimeoutOverrides: - description: HealthTimeoutOverrides allows the internal watchdog timeouts - of individual subcomponents to be overridden. This is useful for - working around "false positive" liveness timeouts that can occur - in particularly stressful workloads or if CPU is constrained. For - a list of active subcomponents, see Felix's logs. - items: - properties: - name: - type: string - timeout: - type: string - required: - - name - - timeout - type: object - type: array - interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' - type: string - interfacePrefix: - description: 'InterfacePrefix is the interface name prefix that identifies - workload endpoints and so distinguishes them from host endpoint - interfaces. Note: in environments other than bare metal, the orchestrators - configure this appropriately. For example our Kubernetes and Docker - integrations set the ''cali'' value, and our OpenStack integration - sets the ''tap'' value. [Default: cali]' - type: string - interfaceRefreshInterval: - description: InterfaceRefreshInterval is the period at which Felix - rescans local interfaces to verify their state. The rescan can be - disabled by setting the interval to 0. - type: string - ipipEnabled: - description: 'IPIPEnabled overrides whether Felix should configure - an IPIP interface on the host. Optional as Felix determines this - based on the existing IP pools. [Default: nil (unset)]' - type: boolean - ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' - type: integer - ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' - type: string - iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is Auto. - type: string - iptablesFilterAllowAction: - type: string - iptablesFilterDenyAction: - description: IptablesFilterDenyAction controls what happens to traffic - that is denied by network policy. By default Calico blocks traffic - with an iptables "DROP" action. If you want to use "REJECT" action - instead you can configure it in here. - type: string - iptablesLockFilePath: - description: 'IptablesLockFilePath is the location of the iptables - lock file. You may need to change this if the lock file is not in - its standard location (for example if you have mapped it into Felix''s - container at a different path). [Default: /run/xtables.lock]' - type: string - iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' - type: string - iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' - type: string - iptablesMangleAllowAction: - type: string - iptablesMarkMask: - description: 'IptablesMarkMask is the mask that Felix selects its - IPTables Mark bits from. Should be a 32 bit hexadecimal number with - at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' - format: int32 - type: integer - iptablesNATOutgoingInterfaceFilter: - type: string - iptablesPostWriteCheckInterval: - description: 'IptablesPostWriteCheckInterval is the period after Felix - has done a write to the dataplane that it schedules an extra read - back in order to check the write was not clobbered by another process. - This should only occur if another application on the system doesn''t - respect the iptables lock. [Default: 1s]' - type: string - iptablesRefreshInterval: - description: 'IptablesRefreshInterval is the period at which Felix - re-checks the IP sets in the dataplane to ensure that no other process - has accidentally broken Calico''s rules. Set to 0 to disable IP - sets refresh. Note: the default for this value is lower than the - other refresh intervals as a workaround for a Linux kernel bug that - was fixed in kernel version 4.11. If you are using v4.11 or greater - you may want to set this to, a higher value to reduce Felix CPU - usage. [Default: 10s]' - type: string - ipv6Support: - description: IPv6Support controls whether Felix enables support for - IPv6 (if supported by the in-use dataplane). - type: boolean - kubeNodePortRanges: - description: 'KubeNodePortRanges holds list of port ranges used for - service node ports. Only used if felix detects kube-proxy running - in ipvs mode. Felix uses these ranges to separate host and workload - traffic. [Default: 30000:32767].' - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - logDebugFilenameRegex: - description: LogDebugFilenameRegex controls which source code files - have their Debug log output included in the logs. Only logs from - files with names that match the given regular expression are included. The - filter only applies to Debug level logs. - type: string - logFilePath: - description: 'LogFilePath is the full path to the Felix log. Set to - none to disable file logging. [Default: /var/log/calico/felix.log]' - type: string - logPrefix: - description: 'LogPrefix is the log prefix that Felix uses when rendering - LOG rules. [Default: calico-packet]' - type: string - logSeverityFile: - description: 'LogSeverityFile is the log severity above which logs - are sent to the log file. [Default: Info]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: Info]' - type: string - logSeveritySys: - description: 'LogSeveritySys is the log severity above which logs - are sent to the syslog. Set to None for no logging to syslog. [Default: - Info]' - type: string - maxIpsetSize: - type: integer - metadataAddr: - description: 'MetadataAddr is the IP address or domain name of the - server that can answer VM queries for cloud-init metadata. In OpenStack, - this corresponds to the machine running nova-api (or in Ubuntu, - nova-api-metadata). A value of none (case insensitive) means that - Felix should not set up any NAT rule for the metadata path. [Default: - 127.0.0.1]' - type: string - metadataPort: - description: 'MetadataPort is the port of the metadata server. This, - combined with global.MetadataAddr (if not ''None''), is used to - set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. - In most cases this should not need to be changed [Default: 8775].' - type: integer - mtuIfacePattern: - description: MTUIfacePattern is a regular expression that controls - which interfaces Felix should scan in order to calculate the host's - MTU. This should not match workload interfaces (usually named cali...). - type: string - natOutgoingAddress: - description: NATOutgoingAddress specifies an address to use when performing - source NAT for traffic in a natOutgoing pool that is leaving the - network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) - type: string - natPortRange: - anyOf: - - type: integer - - type: string - description: NATPortRange specifies the range of ports that is used - for port mapping when doing outgoing NAT. When unset the default - behavior of the network stack is used. - pattern: ^.* - x-kubernetes-int-or-string: true - netlinkTimeout: - type: string - openstackRegion: - description: 'OpenstackRegion is the name of the region that a particular - Felix belongs to. In a multi-region Calico/OpenStack deployment, - this must be configured somehow for each Felix (here in the datamodel, - or in felix.cfg or the environment on each compute node), and must - match the [calico] openstack_region value configured in neutron.conf - on each node. [Default: Empty]' - type: string - policySyncPathPrefix: - description: 'PolicySyncPathPrefix is used to by Felix to communicate - policy changes to external services, like Application layer policy. - [Default: Empty]' - type: string - prometheusGoMetricsEnabled: - description: 'PrometheusGoMetricsEnabled disables Go runtime metrics - collection, which the Prometheus client does by default, when set - to false. This reduces the number of metrics reported, reducing - Prometheus load. [Default: true]' - type: boolean - prometheusMetricsEnabled: - description: 'PrometheusMetricsEnabled enables the Prometheus metrics - server in Felix if set to true. [Default: false]' - type: boolean - prometheusMetricsHost: - description: 'PrometheusMetricsHost is the host that the Prometheus - metrics server should bind to. [Default: empty]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. [Default: 9091]' - type: integer - prometheusProcessMetricsEnabled: - description: 'PrometheusProcessMetricsEnabled disables process metrics - collection, which the Prometheus client does by default, when set - to false. This reduces the number of metrics reported, reducing - Prometheus load. [Default: true]' - type: boolean - prometheusWireGuardMetricsEnabled: - description: 'PrometheusWireGuardMetricsEnabled disables wireguard - metrics collection, which the Prometheus client does by default, - when set to false. This reduces the number of metrics reported, - reducing Prometheus load. [Default: true]' - type: boolean - removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. - type: boolean - reportingInterval: - description: 'ReportingInterval is the interval at which Felix reports - its status into the datastore or 0 to disable. Must be non-zero - in OpenStack deployments. [Default: 30s]' - type: string - reportingTTL: - description: 'ReportingTTL is the time-to-live setting for process-wide - status reports. [Default: 90s]' - type: string - routeRefreshInterval: - description: 'RouteRefreshInterval is the period at which Felix re-checks - the routes in the dataplane to ensure that no other process has - accidentally broken Calico''s rules. Set to 0 to disable route refresh. - [Default: 90s]' - type: string - routeSource: - description: 'RouteSource configures where Felix gets its routing - information. - WorkloadIPs: use workload endpoints to construct - routes. - CalicoIPAM: the default - use IPAM data to construct routes.' - type: string - routeSyncDisabled: - description: RouteSyncDisabled will disable all operations performed - on the route table. Set to true to run in network-policy mode only. - type: boolean - routeTableRange: - description: Deprecated in favor of RouteTableRanges. Calico programs - additional Linux route tables for various purposes. RouteTableRange - specifies the indices of the route tables that Calico should use. - properties: - max: - type: integer - min: - type: integer - required: - - max - - min - type: object - routeTableRanges: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRanges specifies a set of table index ranges - that Calico should use. DeprecatesRouteTableRange, overrides RouteTableRange. - items: - properties: - max: - type: integer - min: - type: integer - required: - - max - - min - type: object - type: array - serviceLoopPrevention: - description: 'When service IP advertisement is enabled, prevent routing - loops to service IPs that are not in use, by dropping or rejecting - packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", - in which case such routing loops continue to be allowed. [Default: - Drop]' - type: string - sidecarAccelerationEnabled: - description: 'SidecarAccelerationEnabled enables experimental sidecar - acceleration [Default: false]' - type: boolean - usageReportingEnabled: - description: 'UsageReportingEnabled reports anonymous Calico version - number and cluster size to projectcalico.org. Logs warnings returned - by the usage server. For example, if a significant security vulnerability - has been discovered in the version of Calico being used. [Default: - true]' - type: boolean - usageReportingInitialDelay: - description: 'UsageReportingInitialDelay controls the minimum delay - before Felix makes a report. [Default: 300s]' - type: string - usageReportingInterval: - description: 'UsageReportingInterval controls the interval at which - Felix makes reports. [Default: 86400s]' - type: string - useInternalDataplaneDriver: - description: UseInternalDataplaneDriver, if true, Felix will use its - internal dataplane programming logic. If false, it will launch - an external dataplane driver and communicate with it over protobuf. - type: boolean - vxlanEnabled: - description: 'VXLANEnabled overrides whether Felix should create the - VXLAN tunnel device for IPv4 VXLAN networking. Optional as Felix - determines this based on the existing IP pools. [Default: nil (unset)]' - type: boolean - vxlanMTU: - description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel - device. See Configuring MTU [Default: 1410]' - type: integer - vxlanMTUV6: - description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel - device. See Configuring MTU [Default: 1390]' - type: integer - vxlanPort: - type: integer - vxlanVNI: - type: integer - wireguardEnabled: - description: 'WireguardEnabled controls whether Wireguard is enabled - for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network). - [Default: false]' - type: boolean - wireguardEnabledV6: - description: 'WireguardEnabledV6 controls whether Wireguard is enabled - for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network). - [Default: false]' - type: boolean - wireguardHostEncryptionEnabled: - description: 'WireguardHostEncryptionEnabled controls whether Wireguard - host-to-host encryption is enabled. [Default: false]' - type: boolean - wireguardInterfaceName: - description: 'WireguardInterfaceName specifies the name to use for - the IPv4 Wireguard interface. [Default: wireguard.cali]' - type: string - wireguardInterfaceNameV6: - description: 'WireguardInterfaceNameV6 specifies the name to use for - the IPv6 Wireguard interface. [Default: wg-v6.cali]' - type: string - wireguardKeepAlive: - description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive - option. Set 0 to disable. [Default: 0]' - type: string - wireguardListeningPort: - description: 'WireguardListeningPort controls the listening port used - by IPv4 Wireguard. [Default: 51820]' - type: integer - wireguardListeningPortV6: - description: 'WireguardListeningPortV6 controls the listening port - used by IPv6 Wireguard. [Default: 51821]' - type: integer - wireguardMTU: - description: 'WireguardMTU controls the MTU on the IPv4 Wireguard - interface. See Configuring MTU [Default: 1440]' - type: integer - wireguardMTUV6: - description: 'WireguardMTUV6 controls the MTU on the IPv6 Wireguard - interface. See Configuring MTU [Default: 1420]' - type: integer - wireguardRoutingRulePriority: - description: 'WireguardRoutingRulePriority controls the priority value - to use for the Wireguard routing rule. [Default: 99]' - type: integer - workloadSourceSpoofing: - description: WorkloadSourceSpoofing controls whether pods can use - the allowedSourcePrefixes annotation to send traffic with a source - IP address that is not theirs. This is disabled by default. When - set to "Any", pods can request any prefix. - type: string - xdpEnabled: - description: 'XDPEnabled enables XDP acceleration for suitable untracked - incoming deny rules. [Default: true]' - type: boolean - xdpRefreshInterval: - description: 'XDPRefreshInterval is the period at which Felix re-checks - all XDP state to ensure that no other process has accidentally broken - Calico''s BPF maps or attached programs. Set to 0 to disable XDP - refresh. [Default: 90s]' - type: string - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: globalnetworkpolicies.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: GlobalNetworkPolicy - listKind: GlobalNetworkPolicyList - plural: globalnetworkpolicies - singular: globalnetworkpolicy - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - applyOnForward: - description: ApplyOnForward indicates to apply the rules in this policy - on forward traffic. - type: boolean - doNotTrack: - description: DoNotTrack indicates whether packets matched by the rules - in this policy should go through the data plane's connection tracking, - such as Linux conntrack. If True, the rules in this policy are - applied before any data plane connection tracking, and packets allowed - by this policy are marked as not to be tracked. - type: boolean - egress: - description: The ordered set of egress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with \"Not\". All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a exact or a prefix match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - ingress: - description: The ordered set of ingress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with \"Not\". All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a exact or a prefix match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - namespaceSelector: - description: NamespaceSelector is an optional field for an expression - used to select a pod based on namespaces. - type: string - order: - description: Order is an optional field that specifies the order in - which the policy is applied. Policies with higher "order" are applied - after those with lower order. If the order is omitted, it may be - considered to be "infinite" - i.e. the policy will be applied last. Policies - with identical order will be applied in alphanumerical order based - on the Policy "Name". - type: number - preDNAT: - description: PreDNAT indicates to apply the rules in this policy before - any DNAT. - type: boolean - selector: - description: "The selector is an expression used to pick pick out - the endpoints that the policy should be applied to. \n Selector - expressions follow this syntax: \n \tlabel == \"string_literal\" - \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" - \ -> not equal; also matches if label is not present \tlabel in - { \"a\", \"b\", \"c\", ... } -> true if the value of label X is - one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", - ... } -> true if the value of label X is not one of \"a\", \"b\", - \"c\" \thas(label_name) -> True if that label is present \t! expr - -> negation of expr \texpr && expr -> Short-circuit and \texpr - || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() - or the empty selector -> matches all endpoints. \n Label names are - allowed to contain alphanumerics, -, _ and /. String literals are - more permissive but they do not support escape characters. \n Examples - (with made-up labels): \n \ttype == \"webserver\" && deployment - == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != - \"dev\" \t! has(label_name)" - type: string - serviceAccountSelector: - description: ServiceAccountSelector is an optional field for an expression - used to select a pod based on service accounts. - type: string - types: - description: "Types indicates whether this policy applies to ingress, - or to egress, or to both. When not explicitly specified (and so - the value on creation is empty or nil), Calico defaults Types according - to what Ingress and Egress rules are present in the policy. The - default is: \n - [ PolicyTypeIngress ], if there are no Egress rules - (including the case where there are also no Ingress rules) \n - - [ PolicyTypeEgress ], if there are Egress rules but no Ingress - rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are - both Ingress and Egress rules. \n When the policy is read back again, - Types will always be one of these values, never empty or nil." - items: - description: PolicyType enumerates the possible values of the PolicySpec - Types field. - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: globalnetworksets.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: GlobalNetworkSet - listKind: GlobalNetworkSetList - plural: globalnetworksets - singular: globalnetworkset - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs - that share labels to allow rules to refer to them via selectors. The labels - of GlobalNetworkSet are not namespaced. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: GlobalNetworkSetSpec contains the specification for a NetworkSet - resource. - properties: - nets: - description: The list of IP networks that belong to this set. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: hostendpoints.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: HostEndpoint - listKind: HostEndpointList - plural: hostendpoints - singular: hostendpoint - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: HostEndpointSpec contains the specification for a HostEndpoint - resource. - properties: - expectedIPs: - description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. - If \"InterfaceName\" is not present, Calico will look for an interface - matching any of the IPs in the list and apply policy to that. Note: - \tWhen using the selector match criteria in an ingress or egress - security Policy \tor Profile, Calico converts the selector into - a set of IP addresses. For host \tendpoints, the ExpectedIPs field - is used for that purpose. (If only the interface \tname is specified, - Calico does not learn the IPs of the interface for use in match - \tcriteria.)" - items: - type: string - type: array - interfaceName: - description: "Either \"*\", or the name of a specific Linux interface - to apply policy to; or empty. \"*\" indicates that this HostEndpoint - governs all traffic to, from or through the default network namespace - of the host named by the \"Node\" field; entering and leaving that - namespace via any interface, including those from/to non-host-networked - local workloads. \n If InterfaceName is not \"*\", this HostEndpoint - only governs traffic that enters or leaves the host through the - specific interface named by InterfaceName, or - when InterfaceName - is empty - through the specific interface that has one of the IPs - in ExpectedIPs. Therefore, when InterfaceName is empty, at least - one expected IP must be specified. Only external interfaces (such - as \"eth0\") are supported here; it isn't possible for a HostEndpoint - to protect traffic through a specific local workload interface. - \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; - initially just pre-DNAT policy. Please check Calico documentation - for the latest position." - type: string - node: - description: The node name identifying the Calico node instance. - type: string - ports: - description: Ports contains the endpoint's named ports, which may - be referenced in security policy rules. - items: - properties: - name: - type: string - port: - type: integer - protocol: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - required: - - name - - port - - protocol - type: object - type: array - profiles: - description: A list of identifiers of security Profile objects that - apply to this endpoint. Each profile is applied in the order that - they appear in this list. Profile rules are applied after the selector-based - security policy. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ipamblocks.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMBlock - listKind: IPAMBlockList - plural: ipamblocks - singular: ipamblock - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMBlockSpec contains the specification for an IPAMBlock - resource. - properties: - affinity: - description: Affinity of the block, if this block has one. If set, - it will be of the form "host:". If not set, this block - is not affine to a host. - type: string - allocations: - description: Array of allocations in-use within this block. nil entries - mean the allocation is free. For non-nil entries at index i, the - index is the ordinal of the allocation within this block and the - value is the index of the associated attributes in the Attributes - array. - items: - type: integer - # TODO: This nullable is manually added in. We should update controller-gen - # to handle []*int properly itself. - nullable: true - type: array - attributes: - description: Attributes is an array of arbitrary metadata associated - with allocations in the block. To find attributes for a given allocation, - use the value of the allocation's entry in the Allocations array - as the index of the element in this array. - items: - properties: - handle_id: - type: string - secondary: - additionalProperties: - type: string - type: object - type: object - type: array - cidr: - description: The block's CIDR. - type: string - deleted: - description: Deleted is an internal boolean used to workaround a limitation - in the Kubernetes API whereby deletion will not return a conflict - error if the block has been updated. It should not be set manually. - type: boolean - sequenceNumber: - default: 0 - description: We store a sequence number that is updated each time - the block is written. Each allocation will also store the sequence - number of the block at the time of its creation. When releasing - an IP, passing the sequence number associated with the allocation - allows us to protect against a race condition and ensure the IP - hasn't been released and re-allocated since the release request. - format: int64 - type: integer - sequenceNumberForAllocation: - additionalProperties: - format: int64 - type: integer - description: Map of allocated ordinal within the block to sequence - number of the block at the time of allocation. Kubernetes does not - allow numerical keys for maps, so the key is cast to a string. - type: object - strictAffinity: - description: StrictAffinity on the IPAMBlock is deprecated and no - longer used by the code. Use IPAMConfig StrictAffinity instead. - type: boolean - unallocated: - description: Unallocated is an ordered list of allocations which are - free in the block. - items: - type: integer - type: array - required: - - allocations - - attributes - - cidr - - strictAffinity - - unallocated - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ipamconfigs.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMConfig - listKind: IPAMConfigList - plural: ipamconfigs - singular: ipamconfig - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMConfigSpec contains the specification for an IPAMConfig - resource. - properties: - autoAllocateBlocks: - type: boolean - maxBlocksPerHost: - description: MaxBlocksPerHost, if non-zero, is the max number of blocks - that can be affine to each host. - maximum: 2147483647 - minimum: 0 - type: integer - strictAffinity: - type: boolean - required: - - autoAllocateBlocks - - strictAffinity - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ipamhandles.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMHandle - listKind: IPAMHandleList - plural: ipamhandles - singular: ipamhandle - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMHandleSpec contains the specification for an IPAMHandle - resource. - properties: - block: - additionalProperties: - type: integer - type: object - deleted: - type: boolean - handleID: - type: string - required: - - block - - handleID - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ippools.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPPool - listKind: IPPoolList - plural: ippools - singular: ippool - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPPoolSpec contains the specification for an IPPool resource. - properties: - allowedUses: - description: AllowedUse controls what the IP pool will be used for. If - not specified or empty, defaults to ["Tunnel", "Workload"] for back-compatibility - items: - type: string - type: array - blockSize: - description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 122 for IPv6. - type: integer - cidr: - description: The pool CIDR. - type: string - disableBGPExport: - description: 'Disable exporting routes from this IP Pool''s CIDR over - BGP. [Default: false]' - type: boolean - disabled: - description: When disabled is true, Calico IPAM will not assign addresses - from this pool. - type: boolean - ipip: - description: 'Deprecated: this field is only used for APIv1 backwards - compatibility. Setting this field is not allowed, this field is - for internal use only.' - properties: - enabled: - description: When enabled is true, ipip tunneling will be used - to deliver packets to destinations within this pool. - type: boolean - mode: - description: The IPIP mode. This can be one of "always" or "cross-subnet". A - mode of "always" will also use IPIP tunneling for routing to - destination IP addresses within this pool. A mode of "cross-subnet" - will only use IPIP tunneling when the destination node is on - a different subnet to the originating node. The default value - (if not specified) is "always". - type: string - type: object - ipipMode: - description: Contains configuration for IPIP tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling - is disabled). - type: string - nat-outgoing: - description: 'Deprecated: this field is only used for APIv1 backwards - compatibility. Setting this field is not allowed, this field is - for internal use only.' - type: boolean - natOutgoing: - description: When natOutgoing is true, packets sent from Calico networked - containers in this pool to destinations outside of this pool will - be masqueraded. - type: boolean - nodeSelector: - description: Allows IPPool to allocate for a specific node by label - selector. - type: string - vxlanMode: - description: Contains configuration for VXLAN tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. VXLAN - tunneling is disabled). - type: string - required: - - cidr - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: ipreservations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPReservation - listKind: IPReservationList - plural: ipreservations - singular: ipreservation - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPReservationSpec contains the specification for an IPReservation - resource. - properties: - reservedCIDRs: - description: ReservedCIDRs is a list of CIDRs and/or IP addresses - that Calico IPAM will exclude from new allocations. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kubecontrollersconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: KubeControllersConfiguration - listKind: KubeControllersConfigurationList - plural: kubecontrollersconfigurations - singular: kubecontrollersconfiguration - preserveUnknownFields: false - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KubeControllersConfigurationSpec contains the values of the - Kubernetes controllers configuration. - properties: - controllers: - description: Controllers enables and configures individual Kubernetes - controllers - properties: - namespace: - description: Namespace enables and configures the namespace controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - node: - description: Node enables and configures the node controller. - Enabled by default, set to nil to disable. - properties: - hostEndpoint: - description: HostEndpoint controls syncing nodes to host endpoints. - Disabled by default, set to nil to disable. - properties: - autoCreate: - description: 'AutoCreate enables automatic creation of - host endpoints for every node. [Default: Disabled]' - type: string - type: object - leakGracePeriod: - description: 'LeakGracePeriod is the period used by the controller - to determine if an IP address has been leaked. Set to 0 - to disable IP garbage collection. [Default: 15m]' - type: string - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - syncLabels: - description: 'SyncLabels controls whether to copy Kubernetes - node labels to Calico nodes. [Default: Enabled]' - type: string - type: object - policy: - description: Policy enables and configures the policy controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - serviceAccount: - description: ServiceAccount enables and configures the service - account controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - workloadEndpoint: - description: WorkloadEndpoint enables and configures the workload - endpoint controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - type: object - debugProfilePort: - description: DebugProfilePort configures the port to serve memory - and cpu profiles on. If not specified, profiling is disabled. - format: int32 - type: integer - etcdV3CompactionPeriod: - description: 'EtcdV3CompactionPeriod is the period between etcdv3 - compaction requests. Set to 0 to disable. [Default: 10m]' - type: string - healthChecks: - description: 'HealthChecks enables or disables support for health - checks [Default: Enabled]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: Info]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. Set to 0 to disable. [Default: 9094]' - type: integer - required: - - controllers - type: object - status: - description: KubeControllersConfigurationStatus represents the status - of the configuration. It's useful for admins to be able to see the actual - config that was applied, which can be modified by environment variables - on the kube-controllers process. - properties: - environmentVars: - additionalProperties: - type: string - description: EnvironmentVars contains the environment variables on - the kube-controllers that influenced the RunningConfig. - type: object - runningConfig: - description: RunningConfig contains the effective config that is running - in the kube-controllers pod, after merging the API resource with - any environment variables. - properties: - controllers: - description: Controllers enables and configures individual Kubernetes - controllers - properties: - namespace: - description: Namespace enables and configures the namespace - controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - node: - description: Node enables and configures the node controller. - Enabled by default, set to nil to disable. - properties: - hostEndpoint: - description: HostEndpoint controls syncing nodes to host - endpoints. Disabled by default, set to nil to disable. - properties: - autoCreate: - description: 'AutoCreate enables automatic creation - of host endpoints for every node. [Default: Disabled]' - type: string - type: object - leakGracePeriod: - description: 'LeakGracePeriod is the period used by the - controller to determine if an IP address has been leaked. - Set to 0 to disable IP garbage collection. [Default: - 15m]' - type: string - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - syncLabels: - description: 'SyncLabels controls whether to copy Kubernetes - node labels to Calico nodes. [Default: Enabled]' - type: string - type: object - policy: - description: Policy enables and configures the policy controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - serviceAccount: - description: ServiceAccount enables and configures the service - account controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - workloadEndpoint: - description: WorkloadEndpoint enables and configures the workload - endpoint controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - type: object - debugProfilePort: - description: DebugProfilePort configures the port to serve memory - and cpu profiles on. If not specified, profiling is disabled. - format: int32 - type: integer - etcdV3CompactionPeriod: - description: 'EtcdV3CompactionPeriod is the period between etcdv3 - compaction requests. Set to 0 to disable. [Default: 10m]' - type: string - healthChecks: - description: 'HealthChecks enables or disables support for health - checks [Default: Enabled]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which - logs are sent to the stdout. [Default: Info]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. Set to 0 to disable. [Default: - 9094]' - type: integer - required: - - controllers - type: object - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: networkpolicies.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: NetworkPolicy - listKind: NetworkPolicyList - plural: networkpolicies - singular: networkpolicy - preserveUnknownFields: false - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - egress: - description: The ordered set of egress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with \"Not\". All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a exact or a prefix match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - ingress: - description: The ordered set of ingress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with \"Not\". All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a exact or a prefix match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - order: - description: Order is an optional field that specifies the order in - which the policy is applied. Policies with higher "order" are applied - after those with lower order. If the order is omitted, it may be - considered to be "infinite" - i.e. the policy will be applied last. Policies - with identical order will be applied in alphanumerical order based - on the Policy "Name". - type: number - selector: - description: "The selector is an expression used to pick pick out - the endpoints that the policy should be applied to. \n Selector - expressions follow this syntax: \n \tlabel == \"string_literal\" - \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" - \ -> not equal; also matches if label is not present \tlabel in - { \"a\", \"b\", \"c\", ... } -> true if the value of label X is - one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", - ... } -> true if the value of label X is not one of \"a\", \"b\", - \"c\" \thas(label_name) -> True if that label is present \t! expr - -> negation of expr \texpr && expr -> Short-circuit and \texpr - || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() - or the empty selector -> matches all endpoints. \n Label names are - allowed to contain alphanumerics, -, _ and /. String literals are - more permissive but they do not support escape characters. \n Examples - (with made-up labels): \n \ttype == \"webserver\" && deployment - == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != - \"dev\" \t! has(label_name)" - type: string - serviceAccountSelector: - description: ServiceAccountSelector is an optional field for an expression - used to select a pod based on service accounts. - type: string - types: - description: "Types indicates whether this policy applies to ingress, - or to egress, or to both. When not explicitly specified (and so - the value on creation is empty or nil), Calico defaults Types according - to what Ingress and Egress are present in the policy. The default - is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including - the case where there are also no Ingress rules) \n - [ PolicyTypeEgress - ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, - PolicyTypeEgress ], if there are both Ingress and Egress rules. - \n When the policy is read back again, Types will always be one - of these values, never empty or nil." - items: - description: PolicyType enumerates the possible values of the PolicySpec - Types field. - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/kdd-crds.yaml -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: networksets.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: NetworkSet - listKind: NetworkSetList - plural: networksets - singular: networkset - preserveUnknownFields: false - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NetworkSetSpec contains the specification for a NetworkSet - resource. - properties: - nets: - description: The list of IP networks that belong to this set. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -# Source: calico/templates/calico-kube-controllers-rbac.yaml -# Include a clusterrole for the kube-controllers component, -# and bind it to the calico-kube-controllers serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -rules: - # Nodes are watched to monitor for deletions. - - apiGroups: [""] - resources: - - nodes - verbs: - - watch - - list - - get - # Pods are watched to check for existence as part of IPAM controller. - - apiGroups: [""] - resources: - - pods - verbs: - - get - - list - - watch - # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ipreservations - verbs: - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - - watch - # Pools are watched to maintain a mapping of blocks to IP pools. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - verbs: - - list - - watch - # kube-controllers manages hostendpoints. - - apiGroups: ["crd.projectcalico.org"] - resources: - - hostendpoints - verbs: - - get - - list - - create - - update - - delete - # Needs access to update clusterinformations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - clusterinformations - verbs: - - get - - list - - create - - update - - watch - # KubeControllersConfiguration is where it gets its config - - apiGroups: ["crd.projectcalico.org"] - resources: - - kubecontrollersconfigurations - verbs: - # read its own config - - get - # create a default if none exists - - create - # update status - - update - # watch for changes - - watch ---- -# Source: calico/templates/calico-node-rbac.yaml -# Include a clusterrole for the calico-node DaemonSet, -# and bind it to the calico-node serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-node -rules: - # Used for creating service account tokens to be used by the CNI plugin - - apiGroups: [""] - resources: - - serviceaccounts/token - resourceNames: - - calico-cni-plugin - verbs: - - create - # The CNI plugin needs to get pods, nodes, and namespaces. - - apiGroups: [""] - resources: - - pods - - nodes - - namespaces - verbs: - - get - # EndpointSlices are used for Service-based network policy rule - # enforcement. - - apiGroups: ["discovery.k8s.io"] - resources: - - endpointslices - verbs: - - watch - - list - - apiGroups: [""] - resources: - - endpoints - - services - verbs: - # Used to discover service IPs for advertisement. - - watch - - list - # Used to discover Typhas. - - get - # Pod CIDR auto-detection on kubeadm needs access to config maps. - - apiGroups: [""] - resources: - - configmaps - verbs: - - get - - apiGroups: [""] - resources: - - nodes/status - verbs: - # Needed for clearing NodeNetworkUnavailable flag. - - patch - # Calico stores some configuration information in node annotations. - - update - # Watch for changes to Kubernetes NetworkPolicies. - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - # Used by Calico for policy information. - - apiGroups: [""] - resources: - - pods - - namespaces - - serviceaccounts - verbs: - - list - - watch - # The CNI plugin patches pods/status. - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - # Calico monitors various CRDs for config. - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - bgpfilters - - globalbgpconfigs - - bgpconfigurations - - ippools - - ipreservations - - ipamblocks - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - networksets - - clusterinformations - - hostendpoints - - blockaffinities - - caliconodestatuses - verbs: - - get - - list - - watch - # Calico must create and update some CRDs on startup. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - - felixconfigurations - - clusterinformations - verbs: - - create - - update - # Calico must update some CRDs. - - apiGroups: [ "crd.projectcalico.org" ] - resources: - - caliconodestatuses - verbs: - - update - # Calico stores some configuration information on the node. - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - watch - # These permissions are only required for upgrade from v2.6, and can - # be removed after upgrade or on fresh installations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - bgpconfigurations - - bgppeers - verbs: - - create - - update - # These permissions are required for Calico CNI to perform IPAM allocations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - # The CNI plugin and calico/node need to be able to create a default - # IPAMConfiguration - - apiGroups: ["crd.projectcalico.org"] - resources: - - ipamconfigs - verbs: - - get - - create - # Block affinities must also be watchable by confd for route aggregation. - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - verbs: - - watch - # The Calico IPAM migration needs to get daemonsets. These permissions can be - # removed if not upgrading from an installation using host-local IPAM. - - apiGroups: ["apps"] - resources: - - daemonsets - verbs: - - get ---- -# Source: calico/templates/calico-node-rbac.yaml -# CNI cluster role -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-cni-plugin -rules: - - apiGroups: [""] - resources: - - pods - - nodes - - namespaces - verbs: - - get - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - - clusterinformations - - ippools - - ipreservations - - ipamconfigs - verbs: - - get - - list - - create - - update - - delete ---- -# Source: calico/templates/calico-kube-controllers-rbac.yaml -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-kube-controllers -subjects: -- kind: ServiceAccount - name: calico-kube-controllers - namespace: kube-system ---- -# Source: calico/templates/calico-node-rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: calico-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system ---- -# Source: calico/templates/calico-node-rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: calico-cni-plugin -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-cni-plugin -subjects: -- kind: ServiceAccount - name: calico-cni-plugin - namespace: kube-system - -{{ if .TyphaEnabled }} ---- -# Source: calico/templates/calico-typha.yaml -# This manifest creates a Service, which will be backed by Calico's Typha daemon. -# Typha sits in between Felix and the API server, reducing Calico's load on the API server. - -apiVersion: v1 -kind: Service -metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha -spec: - ports: - - port: 5473 - protocol: TCP - targetPort: calico-typha - name: calico-typha - selector: - k8s-app: calico-typha - -{{ end }} ---- -# Source: calico/templates/calico-node.yaml -# This manifest installs the calico-node container, as well -# as the CNI plugins and network config on -# each master and worker node in a Kubernetes cluster. -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: calico-node - namespace: kube-system - labels: - k8s-app: calico-node -spec: - selector: - matchLabels: - k8s-app: calico-node - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - template: - metadata: - labels: - k8s-app: calico-node - spec: - nodeSelector: - kubernetes.io/os: linux - hostNetwork: true - tolerations: - # Make sure calico-node gets scheduled on all nodes. - - effect: NoSchedule - operator: Exists - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - serviceAccountName: calico-node - # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force - # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. - terminationGracePeriodSeconds: 0 - priorityClassName: system-node-critical - initContainers: - # This container performs upgrade from host-local IPAM to calico-ipam. - # It can be deleted if this is a fresh installation, or if you have already - # upgraded to use calico-ipam. - - name: upgrade-ipam - image: {{ .CalicoCniImage }} - imagePullPolicy: IfNotPresent - command: ["/opt/cni/bin/calico-ipam", "-upgrade"] - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - volumeMounts: - - mountPath: /var/lib/cni/networks - name: host-local-net-dir - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - securityContext: - privileged: true - # This container installs the CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: {{ .CalicoCniImage }} - imagePullPolicy: IfNotPresent - command: ["/opt/cni/bin/install"] - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # CNI MTU Config variable - - name: CNI_MTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Prevents the container from sleeping forever. - - name: SLEEP - value: "false" - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - securityContext: - privileged: true - # This init container mounts the necessary filesystems needed by the BPF data plane - # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed - # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. - - name: "mount-bpffs" - image: {{ .CalicoNodeImage }} - imagePullPolicy: IfNotPresent - command: ["calico-node", "-init", "-best-effort"] - volumeMounts: - - mountPath: /sys/fs - name: sys-fs - # Bidirectional is required to ensure that the new mount we make at /sys/fs/bpf propagates to the host - # so that it outlives the init container. - mountPropagation: Bidirectional - - mountPath: /var/run/calico - name: var-run-calico - # Bidirectional is required to ensure that the new mount we make at /run/calico/cgroup propagates to the host - # so that it outlives the init container. - mountPropagation: Bidirectional - # Mount /proc/ from host which usually is an init program at /nodeproc. It's needed by mountns binary, - # executed by calico-node, to mount root cgroup2 fs at /run/calico/cgroup to attach CTLB programs correctly. - - mountPath: /nodeproc - name: nodeproc - readOnly: true - securityContext: - privileged: true - containers: - # Runs calico-node container on each Kubernetes node. This - # container programs network policy and routes on each - # host. - - name: calico-node - image: {{ .CalicoNodeImage }} - imagePullPolicy: IfNotPresent - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" -{{ if .TyphaEnabled }} - # Typha support: controlled by the ConfigMap. - - name: FELIX_TYPHAK8SSERVICENAME - valueFrom: - configMapKeyRef: - name: calico-config - key: typha_service_name -{{ end }} - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # Set based on the k8s node name. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # Choose the backend to use. - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s,bgp" - # Auto-detect the BGP IP address. - - name: NODEIP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: IP_AUTODETECTION_METHOD - value: "can-reach=$(NODEIP)" - - name: IP - value: "autodetect" - # Enable IPIP - - name: CALICO_IPV4POOL_IPIP - value: "{{ .IPIPMode }}" - # Enable or Disable VXLAN on the default IP pool. - - name: CALICO_IPV4POOL_VXLAN - value: "{{ .VXLANMode }}" -{{- if .IPV4POOLNATOUTGOING }} - - name: CALICO_IPV4POOL_NAT_OUTGOING - value: "true" -{{- else }} - - name: CALICO_IPV4POOL_NAT_OUTGOING - value: "false" -{{- end }} - # Enable or Disable VXLAN on the default IPv6 IP pool. - - name: CALICO_IPV6POOL_VXLAN - value: "Never" - # Set MTU for tunnel device used if ipip is enabled - - name: FELIX_IPINIPMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Set MTU for the VXLAN tunnel device. - - name: FELIX_VXLANMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Set MTU for the Wireguard tunnel device. - - name: FELIX_WIREGUARDMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu -{{- if .DefaultIPPOOL }} - # The default IPv4 pool to create on startup if none exists. Pod IPs will be - # chosen from this range. Changing this value after installation will have - # no effect. - - name: CALICO_IPV4POOL_CIDR - value: "{{ .KubePodsCIDR }}" - - name: CALICO_IPV4POOL_BLOCK_SIZE - value: "{{ .NodeCidrMaskSize }}" -{{- else }} - - name: NO_DEFAULT_POOLS - value: "true" - - name: CALICO_IPV4POOL_CIDR - value: "" - - name: CALICO_IPV6POOL_CIDR - value: "" -{{- end }} - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPv6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - - name: FELIX_HEALTHENABLED - value: "true" - - name: FELIX_DEVICEROUTESOURCEADDRESS - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - securityContext: - privileged: true - resources: - requests: - cpu: 250m - lifecycle: - preStop: - exec: - command: - - /bin/calico-node - - -shutdown - livenessProbe: - exec: - command: - - /bin/calico-node - - -felix-live - - -bird-live - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - timeoutSeconds: 10 - readinessProbe: - exec: - command: - - /bin/calico-node - - -felix-ready - - -bird-ready - periodSeconds: 10 - timeoutSeconds: 10 - volumeMounts: - # For maintaining CNI plugin API credentials. - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - readOnly: false - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /run/xtables.lock - name: xtables-lock - readOnly: false - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - - name: policysync - mountPath: /var/run/nodeagent - # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the - # parent directory. - - name: bpffs - mountPath: /sys/fs/bpf - - name: cni-log-dir - mountPath: /var/log/calico/cni - readOnly: true - volumes: - # Used by calico-node. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - - name: sys-fs - hostPath: - path: /sys/fs/ - type: DirectoryOrCreate - - name: bpffs - hostPath: - path: /sys/fs/bpf - type: Directory - # mount /proc at /nodeproc to be used by mount-bpffs initContainer to mount root cgroup2 fs. - - name: nodeproc - hostPath: - path: /proc - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d - # Used to access CNI logs. - - name: cni-log-dir - hostPath: - path: /var/log/calico/cni - # Mount in the directory for host-local IPAM allocations. This is - # used when upgrading from host-local to calico-ipam, and can be removed - # if not using the upgrade-ipam init container. - - name: host-local-net-dir - hostPath: - path: /var/lib/cni/networks - # Used to create per-pod Unix Domain Sockets - - name: policysync - hostPath: - type: DirectoryOrCreate - path: /var/run/nodeagent ---- -# Source: calico/templates/calico-kube-controllers.yaml -# See https://github.com/projectcalico/kube-controllers -apiVersion: apps/v1 -kind: Deployment -metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers -spec: - # The controllers can only have a single active instance. - replicas: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers - strategy: - type: Recreate - template: - metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers - spec: - nodeSelector: - kubernetes.io/os: linux - tolerations: - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - effect: NoSchedule - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule - serviceAccountName: calico-kube-controllers - priorityClassName: system-cluster-critical - containers: - - name: calico-kube-controllers - image: {{ .CalicoControllersImage }} - imagePullPolicy: IfNotPresent - env: - # Choose which controllers to run. - - name: ENABLED_CONTROLLERS - value: node - - name: DATASTORE_TYPE - value: kubernetes - livenessProbe: - exec: - command: - - /usr/bin/check-status - - -l - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - timeoutSeconds: 10 - readinessProbe: - exec: - command: - - /usr/bin/check-status - - -r - periodSeconds: 10 - -{{ if .TyphaEnabled }} ---- -# Source: calico/templates/calico-typha.yaml -# This manifest creates a Deployment of Typha to back the above service. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha -spec: - # Number of Typha replicas. To enable Typha, set this to a non-zero value *and* set the - # typha_service_name variable in the calico-config ConfigMap above. - # - # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential - # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In - # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. - replicas: 1 - revisionHistoryLimit: 2 - selector: - matchLabels: - k8s-app: calico-typha - strategy: - rollingUpdate: - # 100% surge allows a complete up-level set of typha instances to start and become ready, - # which in turn allows all the back-level typha instances to start shutting down. This - # means that connections tend to bounce directly from a back-level instance to an up-level - # instance. - maxSurge: 100% - # In case the cluster is unable to schedule extra surge instances, allow at most one instance - # to shut down to make room. You can set this to 0 if you're sure there'll always be enough room to - # schedule extra typha instances during an upgrade (because setting it to 0 blocks shutdown until - # up-level typha instances are online and ready). - maxUnavailable: 1 - type: RollingUpdate - template: - metadata: - labels: - k8s-app: calico-typha - annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' - spec: - nodeSelector: - kubernetes.io/os: linux - hostNetwork: true - # Typha supports graceful shut down, disconnecting clients slowly during the grace period. - # The TYPHA_SHUTDOWNTIMEOUTSECS env var should be kept in sync with this value. - terminationGracePeriodSeconds: 300 - tolerations: - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - # Since Calico can't network a pod until Typha is up, we need to run Typha itself - # as a host-networked pod. - serviceAccountName: calico-node - priorityClassName: system-cluster-critical - # fsGroup allows using projected serviceaccount tokens as described here kubernetes/kubernetes#82573 - securityContext: - fsGroup: 65534 - containers: - - image: {{ .CalicoTyphaImage }} - imagePullPolicy: IfNotPresent - name: calico-typha - ports: - - containerPort: 5473 - name: calico-typha - protocol: TCP - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Enable "info" logging by default. Can be set to "debug" to increase verbosity. - - name: TYPHA_LOGSEVERITYSCREEN - value: "info" - # Disable logging to file and syslog since those don't make sense in Kubernetes. - - name: TYPHA_LOGFILEPATH - value: "none" - - name: TYPHA_LOGSEVERITYSYS - value: "none" - # Monitor the Kubernetes API to find the number of running instances and rebalance - # connections. - - name: TYPHA_CONNECTIONREBALANCINGMODE - value: "kubernetes" - - name: TYPHA_DATASTORETYPE - value: "kubernetes" - - name: TYPHA_HEALTHENABLED - value: "true" - # Set this to the same value as terminationGracePeriodSeconds; it tells Typha how much time - # it has to shut down. - - name: TYPHA_SHUTDOWNTIMEOUTSECS - value: "300" - # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, - # this opens a port on the host, which may need to be secured. - #- name: TYPHA_PROMETHEUSMETRICSENABLED - # value: "true" - #- name: TYPHA_PROMETHEUSMETRICSPORT - # value: "9093" - livenessProbe: - httpGet: - path: /liveness - port: 9098 - host: localhost - periodSeconds: 30 - initialDelaySeconds: 30 - timeoutSeconds: 10 - securityContext: - runAsNonRoot: true - allowPrivilegeEscalation: false - readinessProbe: - httpGet: - path: /readiness - port: 9098 - host: localhost - periodSeconds: 10 - timeoutSeconds: 10 - -{{ end }} - - `))) diff --git a/cmd/kk/pkg/plugins/network/templates/calico_v1.16-.go b/cmd/kk/pkg/plugins/network/templates/calico_v1.16-.go deleted file mode 100644 index dc56d7c6..00000000 --- a/cmd/kk/pkg/plugins/network/templates/calico_v1.16-.go +++ /dev/null @@ -1,1037 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var CalicoOld = template.Must(template.New("network-plugin.yaml").Parse( - dedent.Dedent(`--- -# Source: calico/templates/calico-config.yaml -# This ConfigMap is used to configure a self-hosted Calico installation. -kind: ConfigMap -apiVersion: v1 -metadata: - name: calico-config - namespace: kube-system -data: - # You must set a non-zero value for Typha replicas below. - typha_service_name: {{ if .TyphaEnabled }}"calico-typha"{{ else }}"none"{{ end }} - # Configure the backend to use. - calico_backend: "bird" - # Configure the MTU to use for workload interfaces and the - # tunnels. For IPIP, set to your network MTU - 20; for VXLAN - # set to your network MTU - 50. - veth_mtu: "{{ .VethMTU }}" - - # The CNI network configuration to install on each node. The special - # values in this config will be automatically populated. - cni_network_config: |- - { - "name": "k8s-pod-network", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "calico", - "log_level": "info", - "datastore_type": "kubernetes", - "nodename": "__KUBERNETES_NODE_NAME__", - "mtu": __CNI_MTU__, - "ipam": { - "type": "calico-ipam" - }, - "policy": { - "type": "k8s" - }, - "kubernetes": { - "kubeconfig": "__KUBECONFIG_FILEPATH__" - } - }, - { - "type": "portmap", - "snat": true, - "capabilities": {"portMappings": true} - }, - { - "type": "bandwidth", - "capabilities": {"bandwidth": true} - } - ] - } - ---- -# Source: calico/templates/kdd-crds.yaml - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: bgpconfigurations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: BGPConfiguration - plural: bgpconfigurations - singular: bgpconfiguration - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: bgppeers.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: BGPPeer - plural: bgppeers - singular: bgppeer - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: blockaffinities.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: BlockAffinity - plural: blockaffinities - singular: blockaffinity - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: clusterinformations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: ClusterInformation - plural: clusterinformations - singular: clusterinformation - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: felixconfigurations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: FelixConfiguration - plural: felixconfigurations - singular: felixconfiguration - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: globalnetworkpolicies.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: GlobalNetworkPolicy - plural: globalnetworkpolicies - singular: globalnetworkpolicy - shortNames: - - gnp - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: globalnetworksets.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: GlobalNetworkSet - plural: globalnetworksets - singular: globalnetworkset - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: hostendpoints.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: HostEndpoint - plural: hostendpoints - singular: hostendpoint - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: ipamblocks.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: IPAMBlock - plural: ipamblocks - singular: ipamblock - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: ipamconfigs.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: IPAMConfig - plural: ipamconfigs - singular: ipamconfig - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: ipamhandles.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: IPAMHandle - plural: ipamhandles - singular: ipamhandle - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: ippools.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: IPPool - plural: ippools - singular: ippool - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: kubecontrollersconfigurations.crd.projectcalico.org -spec: - scope: Cluster - group: crd.projectcalico.org - version: v1 - names: - kind: KubeControllersConfiguration - plural: kubecontrollersconfigurations - singular: kubecontrollersconfiguration ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: networkpolicies.crd.projectcalico.org -spec: - scope: Namespaced - group: crd.projectcalico.org - version: v1 - names: - kind: NetworkPolicy - plural: networkpolicies - singular: networkpolicy - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: networksets.crd.projectcalico.org -spec: - scope: Namespaced - group: crd.projectcalico.org - version: v1 - names: - kind: NetworkSet - plural: networksets - singular: networkset - ---- ---- -# Source: calico/templates/rbac.yaml - -# Include a clusterrole for the kube-controllers component, -# and bind it to the calico-kube-controllers serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -rules: - # Nodes are watched to monitor for deletions. - - apiGroups: [""] - resources: - - nodes - verbs: - - watch - - list - - get - # Pods are queried to check for existence. - - apiGroups: [""] - resources: - - pods - verbs: - - get - # IPAM resources are manipulated when nodes are deleted. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - verbs: - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - # kube-controllers manages hostendpoints. - - apiGroups: ["crd.projectcalico.org"] - resources: - - hostendpoints - verbs: - - get - - list - - create - - update - - delete - # Needs access to update clusterinformations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - clusterinformations - verbs: - - get - - create - - update - # KubeControllersConfiguration is where it gets its config - - apiGroups: ["crd.projectcalico.org"] - resources: - - kubecontrollersconfigurations - verbs: - # read its own config - - get - # create a default if none exists - - create - # update status - - update - # watch for changes - - watch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-kube-controllers -subjects: -- kind: ServiceAccount - name: calico-kube-controllers - namespace: kube-system ---- -# Include a clusterrole for the calico-node DaemonSet, -# and bind it to the calico-node serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-node -rules: - # The CNI plugin needs to get pods, nodes, and namespaces. - - apiGroups: [""] - resources: - - pods - - nodes - - namespaces - verbs: - - get - - apiGroups: [""] - resources: - - endpoints - - services - verbs: - # Used to discover service IPs for advertisement. - - watch - - list - # Used to discover Typhas. - - get - # Pod CIDR auto-detection on kubeadm needs access to config maps. - - apiGroups: [""] - resources: - - configmaps - verbs: - - get - - apiGroups: [""] - resources: - - nodes/status - verbs: - # Needed for clearing NodeNetworkUnavailable flag. - - patch - # Calico stores some configuration information in node annotations. - - update - # Watch for changes to Kubernetes NetworkPolicies. - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - # Used by Calico for policy information. - - apiGroups: [""] - resources: - - pods - - namespaces - - serviceaccounts - verbs: - - list - - watch - # The CNI plugin patches pods/status. - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - # Calico monitors various CRDs for config. - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - ipamblocks - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - networksets - - clusterinformations - - hostendpoints - - blockaffinities - verbs: - - get - - list - - watch - # Calico must create and update some CRDs on startup. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - - felixconfigurations - - clusterinformations - verbs: - - create - - update - # Calico stores some configuration information on the node. - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - watch - # These permissions are only requried for upgrade from v2.6, and can - # be removed after upgrade or on fresh installations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - bgpconfigurations - - bgppeers - verbs: - - create - - update - # These permissions are required for Calico CNI to perform IPAM allocations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - - apiGroups: ["crd.projectcalico.org"] - resources: - - ipamconfigs - verbs: - - get - # Block affinities must also be watchable by confd for route aggregation. - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - verbs: - - watch - # The Calico IPAM migration needs to get daemonsets. These permissions can be - # removed if not upgrading from an installation using host-local IPAM. - - apiGroups: ["apps"] - resources: - - daemonsets - verbs: - - get - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: calico-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system - -{{ if .TyphaEnabled }} ---- -# Source: calico/templates/calico-typha.yaml -# This manifest creates a Service, which will be backed by Calico's Typha daemon. -# Typha sits in between Felix and the API server, reducing Calico's load on the API server. - -apiVersion: v1 -kind: Service -metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha -spec: - ports: - - port: 5473 - protocol: TCP - targetPort: calico-typha - name: calico-typha - selector: - k8s-app: calico-typha - ---- - -# This manifest creates a Deployment of Typha to back the above service. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha -spec: - # Number of Typha replicas. To enable Typha, set this to a non-zero value *and* set the - # typha_service_name variable in the calico-config ConfigMap above. - # - # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential - # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In - # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. - replicas: 1 - revisionHistoryLimit: 2 - selector: - matchLabels: - k8s-app: calico-typha - template: - metadata: - labels: - k8s-app: calico-typha - annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' - spec: - nodeSelector: - kubernetes.io/os: linux - hostNetwork: true - tolerations: - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - # Since Calico can't network a pod until Typha is up, we need to run Typha itself - # as a host-networked pod. - serviceAccountName: calico-node - priorityClassName: system-cluster-critical - # fsGroup allows using projected serviceaccount tokens as described here kubernetes/kubernetes#82573 - securityContext: - fsGroup: 65534 - containers: - - image: {{ .CalicoTyphaImage }} - name: calico-typha - ports: - - containerPort: 5473 - name: calico-typha - protocol: TCP - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Enable "info" logging by default. Can be set to "debug" to increase verbosity. - - name: TYPHA_LOGSEVERITYSCREEN - value: "info" - # Disable logging to file and syslog since those don't make sense in Kubernetes. - - name: TYPHA_LOGFILEPATH - value: "none" - - name: TYPHA_LOGSEVERITYSYS - value: "none" - # Monitor the Kubernetes API to find the number of running instances and rebalance - # connections. - - name: TYPHA_CONNECTIONREBALANCINGMODE - value: "kubernetes" - - name: TYPHA_DATASTORETYPE - value: "kubernetes" - - name: TYPHA_HEALTHENABLED - value: "true" - # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, - # this opens a port on the host, which may need to be secured. - #- name: TYPHA_PROMETHEUSMETRICSENABLED - # value: "true" - #- name: TYPHA_PROMETHEUSMETRICSPORT - # value: "9093" - livenessProbe: - httpGet: - path: /liveness - port: 9098 - host: localhost - periodSeconds: 30 - initialDelaySeconds: 30 - securityContext: - runAsNonRoot: true - allowPrivilegeEscalation: false - readinessProbe: - httpGet: - path: /readiness - port: 9098 - host: localhost - periodSeconds: 10 - - --- - - # This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict - - apiVersion: policy/v1beta1 - kind: PodDisruptionBudget - metadata: - name: calico-typha - namespace: kube-system - labels: - k8s-app: calico-typha - spec: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: calico-typha - {{ end }} - ---- -# Source: calico/templates/calico-node.yaml -# This manifest installs the calico-node container, as well -# as the CNI plugins and network config on -# each master and worker node in a Kubernetes cluster. -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: calico-node - namespace: kube-system - labels: - k8s-app: calico-node -spec: - selector: - matchLabels: - k8s-app: calico-node - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - template: - metadata: - labels: - k8s-app: calico-node - spec: - nodeSelector: - kubernetes.io/os: linux - hostNetwork: true - tolerations: - # Make sure calico-node gets scheduled on all nodes. - - effect: NoSchedule - operator: Exists - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - serviceAccountName: calico-node - # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force - # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. - terminationGracePeriodSeconds: 0 - priorityClassName: system-node-critical - initContainers: - # This container performs upgrade from host-local IPAM to calico-ipam. - # It can be deleted if this is a fresh installation, or if you have already - # upgraded to use calico-ipam. - - name: upgrade-ipam - image: {{ .CalicoCniImage }} - command: ["/opt/cni/bin/calico-ipam", "-upgrade"] - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - volumeMounts: - - mountPath: /var/lib/cni/networks - name: host-local-net-dir - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - securityContext: - privileged: true - # This container installs the CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: {{ .CalicoCniImage }} - command: ["/opt/cni/bin/install"] - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # CNI MTU Config variable - - name: CNI_MTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Prevents the container from sleeping forever. - - name: SLEEP - value: "false" - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - securityContext: - privileged: true - # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes - # to communicate with Felix over the Policy Sync API. - - name: flexvol-driver - image: {{ .CalicoFlexvolImage }} - volumeMounts: - - name: flexvol-driver-host - mountPath: /host/driver - securityContext: - privileged: true - containers: - # Runs calico-node container on each Kubernetes node. This - # container programs network policy and routes on each - # host. - - name: calico-node - image: {{ .CalicoNodeImage }} - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" -{{ if .TyphaEnabled }} - # Typha support: controlled by the ConfigMap. - - name: FELIX_TYPHAK8SSERVICENAME - valueFrom: - configMapKeyRef: - name: calico-config - key: typha_service_name -{{ end }} - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # Set based on the k8s node name. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # Choose the backend to use. - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s,bgp" - # Auto-detect the BGP IP address. - - name: NODEIP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: IP_AUTODETECTION_METHOD - value: "can-reach=$(NODEIP)" - - name: IP - value: "autodetect" - # Enable IPIP - - name: CALICO_IPV4POOL_IPIP - value: "{{ .IPIPMode }}" - # Enable or Disable VXLAN on the default IP pool. - - name: CALICO_IPV4POOL_VXLAN - value: "{{ .VXLANMode }}" - # Set MTU for tunnel device used if ipip is enabled - - name: FELIX_IPINIPMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Set MTU for the VXLAN tunnel device. - - name: FELIX_VXLANMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Set MTU for the Wireguard tunnel device. - - name: FELIX_WIREGUARDMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # The default IPv4 pool to create on startup if none exists. Pod IPs will be - # chosen from this range. Changing this value after installation will have - # no effect. - - name: CALICO_IPV4POOL_CIDR - value: "{{ .KubePodsCIDR }}" - - name: CALICO_IPV4POOL_BLOCK_SIZE - value: "{{ .NodeCidrMaskSize }}" - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPv6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - # Set Felix logging to "info" - - name: FELIX_LOGSEVERITYSCREEN - value: "info" - - name: FELIX_HEALTHENABLED - value: "true" - - name: FELIX_DEVICEROUTESOURCEADDRESS - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - securityContext: - privileged: true - resources: - requests: - cpu: 250m - livenessProbe: - exec: - command: - - /bin/calico-node - - -felix-live - - -bird-live - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - readinessProbe: - exec: - command: - - /bin/calico-node - - -felix-ready - - -bird-ready - periodSeconds: 10 - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /run/xtables.lock - name: xtables-lock - readOnly: false - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - - name: policysync - mountPath: /var/run/nodeagent - # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the - # parent directory. - - name: sysfs - mountPath: /sys/fs/ - # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host. - # If the host is known to mount that filesystem already then Bidirectional can be omitted. - mountPropagation: Bidirectional - volumes: - # Used by calico-node. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - - name: sysfs - hostPath: - path: /sys/fs/ - type: DirectoryOrCreate - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d - # Mount in the directory for host-local IPAM allocations. This is - # used when upgrading from host-local to calico-ipam, and can be removed - # if not using the upgrade-ipam init container. - - name: host-local-net-dir - hostPath: - path: /var/lib/cni/networks - # Used to create per-pod Unix Domain Sockets - - name: policysync - hostPath: - type: DirectoryOrCreate - path: /var/run/nodeagent - # Used to install Flex Volume Driver - - name: flexvol-driver-host - hostPath: - type: DirectoryOrCreate - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system - ---- -# Source: calico/templates/calico-kube-controllers.yaml -# See https://github.com/projectcalico/kube-controllers -apiVersion: apps/v1 -kind: Deployment -metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers -spec: - # The controllers can only have a single active instance. - replicas: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers - strategy: - type: Recreate - template: - metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers - spec: - nodeSelector: - kubernetes.io/os: linux - tolerations: - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - effect: NoSchedule - serviceAccountName: calico-kube-controllers - priorityClassName: system-cluster-critical - containers: - - name: calico-kube-controllers - image: {{ .CalicoControllersImage }} - env: - # Choose which controllers to run. - - name: ENABLED_CONTROLLERS - value: node - - name: DATASTORE_TYPE - value: kubernetes - readinessProbe: - exec: - command: - - /usr/bin/check-status - - -r - ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-kube-controllers - namespace: kube-system - - `))) diff --git a/cmd/kk/pkg/plugins/network/templates/cilium.go b/cmd/kk/pkg/plugins/network/templates/cilium.go deleted file mode 100644 index ba7f6870..00000000 --- a/cmd/kk/pkg/plugins/network/templates/cilium.go +++ /dev/null @@ -1,688 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var Cilium = template.Must(template.New("network-plugin.yaml").Parse( - dedent.Dedent(`--- -# Source: cilium/charts/agent/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cilium - namespace: kube-system ---- -# Source: cilium/charts/operator/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cilium-operator - namespace: kube-system ---- -# Source: cilium/charts/config/templates/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: cilium-config - namespace: kube-system -data: - - # Identity allocation mode selects how identities are shared between cilium - # nodes by setting how they are stored. The options are "crd" or "kvstore". - # - "crd" stores identities in kubernetes as CRDs (custom resource definition). - # These can be queried with: - # kubectl get ciliumid - # - "kvstore" stores identities in a kvstore, etcd or consul, that is - # configured below. Cilium versions before 1.6 supported only the kvstore - # backend. Upgrades from these older cilium versions should continue using - # the kvstore by commenting out the identity-allocation-mode below, or - # setting it to "kvstore". - identity-allocation-mode: crd - - # If you want to run cilium in debug mode change this value to true - debug: "false" - - # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 - # address. - enable-ipv4: "true" - - # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 - # address. - enable-ipv6: "false" - enable-bpf-clock-probe: "true" - - # If you want cilium monitor to aggregate tracing for packets, set this level - # to "low", "medium", or "maximum". The higher the level, the less packets - # that will be seen in monitor output. - monitor-aggregation: medium - - # The monitor aggregation interval governs the typical time between monitor - # notification events for each allowed connection. - # - # Only effective when monitor aggregation is set to "medium" or higher. - monitor-aggregation-interval: 5s - - # The monitor aggregation flags determine which TCP flags which, upon the - # first observation, cause monitor notifications to be generated. - # - # Only effective when monitor aggregation is set to "medium" or higher. - monitor-aggregation-flags: all - # bpf-policy-map-max specified the maximum number of entries in endpoint - # policy map (per endpoint) - bpf-policy-map-max: "16384" - # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic - # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. - bpf-map-dynamic-size-ratio: "0.0025" - - # Pre-allocation of map entries allows per-packet latency to be reduced, at - # the expense of up-front memory allocation for the entries in the maps. The - # default value below will minimize memory usage in the default installation; - # users who are sensitive to latency may consider setting this to "true". - # - # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore - # this option and behave as though it is set to "true". - # - # If this value is modified, then during the next Cilium startup the restore - # of existing endpoints and tracking of ongoing connections may be disrupted. - # This may lead to policy drops or a change in loadbalancing decisions for a - # connection for some time. Endpoints may need to be recreated to restore - # connectivity. - # - # If this option is set to "false" during an upgrade from 1.3 or earlier to - # 1.4 or later, then it may cause one-time disruptions during the upgrade. - preallocate-bpf-maps: "false" - - # Regular expression matching compatible Istio sidecar istio-proxy - # container image names - sidecar-istio-proxy-image: "cilium/istio_proxy" - - # Encapsulation mode for communication between nodes - # Possible values: - # - disabled - # - vxlan (default) - # - geneve - tunnel: vxlan - - # Name of the cluster. Only relevant when building a mesh of clusters. - cluster-name: default - - # wait-bpf-mount makes init container wait until bpf filesystem is mounted - wait-bpf-mount: "false" - - masquerade: "true" - enable-bpf-masquerade: "true" - enable-xt-socket-fallback: "true" - install-iptables-rules: "true" - auto-direct-node-routes: "false" - kube-proxy-replacement: "probe" - enable-health-check-nodeport: "true" - node-port-bind-protection: "true" - enable-auto-protect-node-port-range: "true" - enable-session-affinity: "true" - k8s-require-ipv4-pod-cidr: "true" - k8s-require-ipv6-pod-cidr: "false" - enable-endpoint-health-checking: "true" - enable-well-known-identities: "false" - enable-remote-node-identity: "true" - operator-api-serve-addr: "127.0.0.1:9234" - ipam: "cluster-pool" - cluster-pool-ipv4-cidr: "{{ .KubePodsCIDR }}" - cluster-pool-ipv4-mask-size: "{{ .NodeCidrMaskSize }}" - disable-cnp-status-updates: "true" ---- -# Source: cilium/charts/agent/templates/clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cilium -rules: -- apiGroups: - - networking.k8s.io - resources: - - networkpolicies - verbs: - - get - - list - - watch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - namespaces - - services - - nodes - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods - - nodes - verbs: - - get - - list - - watch - - update -- apiGroups: - - "" - resources: - - nodes - - nodes/status - verbs: - - patch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - create - - get - - list - - watch - - update -- apiGroups: - - cilium.io - resources: - - ciliumnetworkpolicies - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies - - ciliumclusterwidenetworkpolicies/status - - ciliumendpoints - - ciliumendpoints/status - - ciliumnodes - - ciliumnodes/status - - ciliumidentities -# deprecated remove in v1.9 - - ciliumidentities/status - verbs: - - '*' ---- -# Source: cilium/charts/operator/templates/clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cilium-operator -rules: -- apiGroups: - - "" - resources: - # to automatically delete [core|kube]dns pods so that are starting to being - # managed by Cilium - - pods - verbs: - - get - - list - - watch - - delete -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - # to perform the translation of a CNP that contains ToGroup to its endpoints - - services - - endpoints - # to check apiserver connectivity - - namespaces - verbs: - - get - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumnetworkpolicies - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies - - ciliumclusterwidenetworkpolicies/status - - ciliumendpoints - - ciliumendpoints/status - - ciliumnodes - - ciliumnodes/status - - ciliumidentities - - ciliumidentities/status - verbs: - - '*' -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - get - - list - - watch -# For cilium-operator running in HA mode. -# -# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election -# between mulitple running instances. -# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less -# common and fewer objects in the cluster watch "all Leases". -# The support for leases was introduced in coordination.k8s.io/v1 during Kubernetes 1.14 release. -# In Cilium we currently don't support HA mode for K8s version < 1.14. This condition make sure -# that we only authorize access to leases resources in supported K8s versions. -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - update ---- -# Source: cilium/charts/agent/templates/clusterrolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cilium -subjects: -- kind: ServiceAccount - name: cilium - namespace: kube-system ---- -# Source: cilium/charts/operator/templates/clusterrolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cilium-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cilium-operator -subjects: -- kind: ServiceAccount - name: cilium-operator - namespace: kube-system ---- -# Source: cilium/charts/agent/templates/daemonset.yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - k8s-app: cilium - name: cilium - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: cilium - template: - metadata: - annotations: - # This annotation plus the CriticalAddonsOnly toleration makes - # cilium to be a critical pod in the cluster, which ensures cilium - # gets priority scheduling. - # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ - scheduler.alpha.kubernetes.io/critical-pod: "" - labels: - k8s-app: cilium - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchExpressions: - - key: io.cilium/app - operator: In - values: - - operator - topologyKey: kubernetes.io/hostname - weight: 100 - containers: - - args: - - --config-dir=/tmp/cilium/config-map - command: - - cilium-agent - livenessProbe: - httpGet: - host: '127.0.0.1' - path: /healthz - port: 9876 - scheme: HTTP - httpHeaders: - - name: "brief" - value: "true" - failureThreshold: 10 - # The initial delay for the liveness probe is intentionally large to - # avoid an endless kill & restart cycle if in the event that the initial - # bootstrapping takes longer than expected. - initialDelaySeconds: 120 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 5 - readinessProbe: - httpGet: - host: '127.0.0.1' - path: /healthz - port: 9876 - scheme: HTTP - httpHeaders: - - name: "brief" - value: "true" - failureThreshold: 3 - initialDelaySeconds: 5 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 5 - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: CILIUM_FLANNEL_MASTER_DEVICE - valueFrom: - configMapKeyRef: - key: flannel-master-device - name: cilium-config - optional: true - - name: CILIUM_FLANNEL_UNINSTALL_ON_EXIT - valueFrom: - configMapKeyRef: - key: flannel-uninstall-on-exit - name: cilium-config - optional: true - - name: CILIUM_CLUSTERMESH_CONFIG - value: /var/lib/cilium/clustermesh/ - - name: CILIUM_CNI_CHAINING_MODE - valueFrom: - configMapKeyRef: - key: cni-chaining-mode - name: cilium-con - optional: true - - name: CILIUM_CUSTOM_CNI_CONF - valueFrom: - configMapKeyRef: - key: custom-cni-conf - name: cilium-config - optional: true - image: "{{ .CiliumImage }}" - imagePullPolicy: IfNotPresent - lifecycle: - postStart: - exec: - command: - - "/cni-install.sh" - - "--enable-debug=false" - preStop: - exec: - command: - - /cni-uninstall.sh - name: cilium-agent - securityContext: - capabilities: - add: - - NET_ADMIN - - SYS_MODULE - privileged: true - volumeMounts: - - mountPath: /sys/fs/bpf - name: bpf-maps - - mountPath: /var/run/cilium - name: cilium-run - - mountPath: /host/opt/cni/bin - name: cni-path - - mountPath: /host/etc/cni/net.d - name: etc-cni-netd - - mountPath: /var/lib/cilium/clustermesh - name: clustermesh-secrets - readOnly: true - - mountPath: /tmp/cilium/config-map - name: cilium-config-path - readOnly: true - # Needed to be able to load kernel modules - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /run/xtables.lock - name: xtables-lock - hostNetwork: true - initContainers: - - command: - - /init-container.sh - env: - - name: CILIUM_ALL_STATE - valueFrom: - configMapKeyRef: - key: clean-cilium-state - name: cilium-config - optional: true - - name: CILIUM_BPF_STATE - valueFrom: - configMapKeyRef: - key: clean-cilium-bpf-state - name: cilium-config - optional: true - - name: CILIUM_WAIT_BPF_MOUNT - valueFrom: - configMapKeyRef: - key: wait-bpf-mount - name: cilium-config - optional: true - image: "{{ .CiliumImage }}" - imagePullPolicy: IfNotPresent - name: clean-cilium-state - securityContext: - capabilities: - add: - - NET_ADMIN - privileged: true - volumeMounts: - - mountPath: /sys/fs/bpf - name: bpf-maps - mountPropagation: HostToContainer - - mountPath: /var/run/cilium - name: cilium-run - resources: - requests: - cpu: 100m - memory: 100Mi - restartPolicy: Always - priorityClassName: system-node-critical - serviceAccount: cilium - serviceAccountName: cilium - terminationGracePeriodSeconds: 1 - tolerations: - - operator: Exists - volumes: - # To keep state between restarts / upgrades - - hostPath: - path: /var/run/cilium - type: DirectoryOrCreate - name: cilium-run - # To keep state between restarts / upgrades for bpf maps - - hostPath: - path: /sys/fs/bpf - type: DirectoryOrCreate - name: bpf-maps - # To install cilium cni plugin in the host - - hostPath: - path: /opt/cni/bin - type: DirectoryOrCreate - name: cni-path - # To install cilium cni configuration in the host - - hostPath: - path: /etc/cni/net.d - type: DirectoryOrCreate - name: etc-cni-netd - # To be able to load kernel modules - - hostPath: - path: /lib/modules - name: lib-modules - # To access iptables concurrently with other processes (e.g. kube-proxy) - - hostPath: - path: /run/xtables.lock - type: FileOrCreate - name: xtables-lock - # To read the clustermesh configuration - - name: clustermesh-secrets - secret: - defaultMode: 420 - optional: true - secretName: cilium-clustermesh - # To read the configuration from the config map - - configMap: - name: cilium-config - name: cilium-config-path - updateStrategy: - rollingUpdate: - maxUnavailable: 2 - type: RollingUpdate ---- -# Source: cilium/charts/operator/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - io.cilium/app: operator - name: cilium-operator - name: cilium-operator - namespace: kube-system -spec: - # We support HA mode only for Kubernetes version > 1.14 - # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go - # for more details. - replicas: 2 - selector: - matchLabels: - io.cilium/app: operator - name: cilium-operator - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: RollingUpdate - template: - metadata: - annotations: - labels: - io.cilium/app: operator - name: cilium-operator - spec: - # In HA mode, cilium-operator pods must not be scheduled on the same - # node as they will clash with each other. - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchExpressions: - - key: io.cilium/app - operator: In - values: - - operator - topologyKey: kubernetes.io/hostname - weight: 100 - containers: - - args: - - --config-dir=/tmp/cilium/config-map - - --debug=$(CILIUM_DEBUG) - command: - - cilium-operator-generic - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: CILIUM_DEBUG - valueFrom: - configMapKeyRef: - key: debug - name: cilium-config - optional: true - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - key: AWS_ACCESS_KEY_ID - name: cilium-aws - optional: true - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - key: AWS_SECRET_ACCESS_KEY - name: cilium-aws - optional: true - - name: AWS_DEFAULT_REGION - valueFrom: - secretKeyRef: - key: AWS_DEFAULT_REGION - name: cilium-aws - optional: true - image: "{{ .OperatorGenericImage }}" - imagePullPolicy: IfNotPresent - name: cilium-operator - livenessProbe: - httpGet: - host: '127.0.0.1' - path: /healthz - port: 9234 - scheme: HTTP - initialDelaySeconds: 60 - periodSeconds: 10 - timeoutSeconds: 3 - volumeMounts: - - mountPath: /tmp/cilium/config-map - name: cilium-config-path - readOnly: true - hostNetwork: true - restartPolicy: Always - priorityClassName: system-cluster-critical - serviceAccount: cilium-operator - serviceAccountName: cilium-operator - tolerations: - - operator: Exists - volumes: - # To read the configuration from the config map - - configMap: - name: cilium-config - name: cilium-config-path - `))) diff --git a/cmd/kk/pkg/plugins/network/templates/flannel.go b/cmd/kk/pkg/plugins/network/templates/flannel.go deleted file mode 100644 index 560b1682..00000000 --- a/cmd/kk/pkg/plugins/network/templates/flannel.go +++ /dev/null @@ -1,504 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var FlannelPSP = template.Must(template.New("network-plugin.yaml").Parse( - dedent.Dedent(`--- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: psp.flannel.unprivileged - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default - seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default - apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default - apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default -spec: - privileged: false - volumes: - - configMap - - secret - - emptyDir - - hostPath - allowedHostPaths: - - pathPrefix: "/etc/cni/net.d" - - pathPrefix: "/etc/kube-flannel" - - pathPrefix: "/run/flannel" - readOnlyRootFilesystem: false - # Users and groups - runAsUser: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - fsGroup: - rule: RunAsAny - # Privilege Escalation - allowPrivilegeEscalation: false - defaultAllowPrivilegeEscalation: false - # Capabilities - allowedCapabilities: ['NET_ADMIN'] - defaultAddCapabilities: [] - requiredDropCapabilities: [] - # Host namespaces - hostPID: false - hostIPC: false - hostNetwork: true - hostPorts: - - min: 0 - max: 65535 - # SELinux - seLinux: - # SELinux is unused in CaaSP - rule: 'RunAsAny' ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - labels: - k8s-app: flannel - name: flannel -rules: - - apiGroups: ['extensions'] - resources: ['podsecuritypolicies'] - verbs: ['use'] - resourceNames: ['psp.flannel.unprivileged'] - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - networking.k8s.io - resources: - - clustercidrs - verbs: - - list - - watch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - labels: - k8s-app: flannel - name: flannel -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flannel -subjects: -- kind: ServiceAccount - name: flannel - namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flannel - namespace: kube-system ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: kube-flannel-cfg - namespace: kube-system - labels: - tier: node - app: flannel - k8s-app: flannel -data: - cni-conf.json: | - { - "name": "cbr0", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "flannel", - "delegate": { - "hairpinMode": true, - "isDefaultGateway": true - } - }, - { - "type": "portmap", - "capabilities": { - "portMappings": true - } - } - ] - } - net-conf.json: | - { - "Network": "{{ .KubePodsCIDR }}", - "Backend": { - "Type": "{{ .BackendMode }}" - } - } ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-flannel-ds - namespace: kube-system - labels: - tier: node - app: flannel - k8s-app: flannel -spec: - selector: - matchLabels: - app: flannel - k8s-app: flannel - template: - metadata: - labels: - tier: node - app: flannel - k8s-app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - linux - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - priorityClassName: system-node-critical - serviceAccountName: flannel - initContainers: - - name: install-cni-plugin - args: - - -f - - /flannel - - /opt/cni/bin/flannel - command: - - cp - image: {{ .FlannelPluginImage }} - volumeMounts: - - mountPath: /opt/cni/bin - name: cni-plugin - - name: install-cni - image: {{ .FlannelImage }} - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: {{ .FlannelImage }} - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN", "NET_RAW"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - - mountPath: /run/xtables.lock - name: xtables-lock - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni-plugin - hostPath: - path: /opt/cni/bin - - name: cni - hostPath: - path: /etc/cni/net.d - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - - `))) - -var FlannelPS = template.Must(template.New("network-plugin.yaml").Parse( - dedent.Dedent(`--- -apiVersion: v1 -kind: Namespace -metadata: - name: kube-flannel - labels: - pod-security.kubernetes.io/enforce: privileged ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - labels: - k8s-app: flannel - name: flannel -rules: - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - networking.k8s.io - resources: - - clustercidrs - verbs: - - list - - watch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - labels: - k8s-app: flannel - name: flannel -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flannel -subjects: -- kind: ServiceAccount - name: flannel - namespace: kube-flannel ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flannel - namespace: kube-flannel ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: kube-flannel-cfg - namespace: kube-flannel - labels: - tier: node - app: flannel - k8s-app: flannel -data: - cni-conf.json: | - { - "name": "cbr0", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "flannel", - "delegate": { - "hairpinMode": true, - "isDefaultGateway": true - } - }, - { - "type": "portmap", - "capabilities": { - "portMappings": true - } - } - ] - } - net-conf.json: | - { - "Network": "{{ .KubePodsCIDR }}", - "Backend": { - "Type": "{{ .BackendMode }}" - } - } ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-flannel-ds - namespace: kube-flannel - labels: - tier: node - app: flannel - k8s-app: flannel -spec: - selector: - matchLabels: - app: flannel - k8s-app: flannel - template: - metadata: - labels: - tier: node - app: flannel - k8s-app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - linux - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - priorityClassName: system-node-critical - serviceAccountName: flannel - initContainers: - - name: install-cni-plugin - args: - - -f - - /flannel - - /opt/cni/bin/flannel - command: - - cp - image: {{ .FlannelPluginImage }} - volumeMounts: - - mountPath: /opt/cni/bin - name: cni-plugin - - name: install-cni - image: {{ .FlannelImage }} - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: {{ .FlannelImage }} - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN", "NET_RAW"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - - mountPath: /run/xtables.lock - name: xtables-lock - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni-plugin - hostPath: - path: /opt/cni/bin - - name: cni - hostPath: - path: /etc/cni/net.d - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - - `))) diff --git a/cmd/kk/pkg/plugins/network/templates/hybridnet.go b/cmd/kk/pkg/plugins/network/templates/hybridnet.go deleted file mode 100644 index 5a39e9cb..00000000 --- a/cmd/kk/pkg/plugins/network/templates/hybridnet.go +++ /dev/null @@ -1,79 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils" - "github.com/lithammer/dedent" - "text/template" -) - -var HybridnetNetworks = template.Must(template.New("hybridnet-networks.yaml").Funcs(utils.FuncMap).Parse( - dedent.Dedent(` -{{- range $index, $network := .Networks }} ---- -apiVersion: networking.alibaba.com/v1 -kind: Network -metadata: - name: {{ $network.Name }} -spec: -{{- if $network.NetID }} - netID: {{ $network.NetID }} -{{- end }} - type: {{ $network.Type }} -{{- if $network.Mode }} - mode: {{ $network.Mode }} -{{- end }} -{{- if $network.NodeSelector }} - nodeSelector: -{{ toYaml $network.NodeSelector | indent 4 }} -{{- end }} - -{{- range $network.Subnets }} ---- -apiVersion: networking.alibaba.com/v1 -kind: Subnet -metadata: - name: {{ .Name }} -spec: - network: {{ $network.Name }} -{{- if .NetID }} - netID: {{ .NetID }} -{{- end }} - range: - version: "4" - cidr: "{{ .CIDR }}" -{{- if .Gateway }} - gateway: "{{ .Gateway }}" -{{- end }} -{{- if .Start}} - start: "{{ .Start }}" -{{- end}} -{{- if .End}} - end: "{{ .End }}" -{{- end }} -{{- if .ReservedIPs }} - reservedIPs: -{{ toYaml .ReservedIPs | indent 4 }} -{{- end }} -{{- if .ExcludeIPs }} - excludeIPs: -{{ toYaml .ExcludeIPs | indent 4 }} -{{- end }} -{{- end }} -{{- end }} - `))) diff --git a/cmd/kk/pkg/plugins/network/templates/kubectl_ko.go b/cmd/kk/pkg/plugins/network/templates/kubectl_ko.go deleted file mode 100644 index 4a51a9a3..00000000 --- a/cmd/kk/pkg/plugins/network/templates/kubectl_ko.go +++ /dev/null @@ -1,922 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var KubectlKo = template.Must(template.New("kubectl-ko").Parse( - dedent.Dedent(`#!/bin/bash -set -euo pipefail - -KUBE_OVN_NS=kube-system -WITHOUT_KUBE_PROXY=false -OVN_NB_POD= -OVN_SB_POD= -KUBE_OVN_VERSION= -REGISTRY="kubeovn" - -showHelp(){ - echo "kubectl ko {subcommand} [option...]" - echo "Available Subcommands:" - echo " [nb|sb] [status|kick|backup|dbstatus|restore] ovn-db operations show cluster status, kick stale server, backup database, get db consistency status or restore ovn nb db when met 'inconsistent data' error" - echo " nbctl [ovn-nbctl options ...] invoke ovn-nbctl" - echo " sbctl [ovn-sbctl options ...] invoke ovn-sbctl" - echo " vsctl {nodeName} [ovs-vsctl options ...] invoke ovs-vsctl on the specified node" - echo " ofctl {nodeName} [ovs-ofctl options ...] invoke ovs-ofctl on the specified node" - echo " dpctl {nodeName} [ovs-dpctl options ...] invoke ovs-dpctl on the specified node" - echo " appctl {nodeName} [ovs-appctl options ...] invoke ovs-appctl on the specified node" - echo " tcpdump {namespace/podname} [tcpdump options ...] capture pod traffic" - echo " trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp or udp port] trace ovn microflow of specific packet" - echo " diagnose {all|node} [nodename] diagnose connectivity of all nodes or a specific node" - echo " tuning {install-fastpath|local-install-fastpath|remove-fastpath|install-stt|local-install-stt|remove-stt} {centos7|centos8}} [kernel-devel-version] deploy kernel optimisation components to the system" - echo " reload restart all kube-ovn components" - echo " env-check check the environment configuration" -} - -# usage: ipv4_to_hex 192.168.0.1 -ipv4_to_hex(){ - printf "%02x" ${1//./ } -} - -# convert hex to dec (portable version) -hex2dec(){ - for i in $(echo "$@"); do - printf "%d\n" "$(( 0x$i ))" - done -} - -# https://github.com/chmduquesne/wg-ip -# usage: expand_ipv6 2001::1 -expand_ipv6(){ - local ip=$1 - - # prepend 0 if we start with : - echo $ip | grep -qs "^:" && ip="0${ip}" - - # expand :: - if echo $ip | grep -qs "::"; then - local colons=$(echo $ip | sed 's/[^:]//g') - local missing=$(echo ":::::::::" | sed "s/$colons//") - local expanded=$(echo $missing | sed 's/:/:0/g') - ip=$(echo $ip | sed "s/::/$expanded/") - fi - - local blocks=$(echo $ip | grep -o "[0-9a-f]\+") - set $blocks - - printf "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n" \ - $(hex2dec $@) -} - -# convert an IPv6 address to bytes -ipv6_bytes(){ - for x in $(expand_ipv6 $1 | tr ':' ' '); do - printf "%d %d " $((0x$x >> 8 & 0xff)) $((0x$x & 0xff)) - done - echo -} - -# usage: ipIsInCidr 192.168.0.1 192.168.0.0/24 -# return: 0 for true, 1 for false -ipIsInCidr(){ - local ip=$1 - local cidr=$2 - - if [[ $ip =~ .*:.* ]]; then - # IPv6 - cidr=${cidr#*,} - local network=${cidr%/*} - local prefix=${cidr#*/} - local ip_bytes=($(ipv6_bytes $ip)) - local network_bytes=($(ipv6_bytes $network)) - for ((i=0; i<${#ip_bytes[*]}; i++)); do - if [ ${ip_bytes[$i]} -eq ${network_bytes[$i]} ]; then - continue - fi - - if [ $((($i+1)*8)) -le $prefix ]; then - return 1 - fi - if [ $(($i*8)) -ge $prefix ]; then - return 0 - fi - if [ $((($i+1)*8)) -le $prefix ]; then - return 1 - fi - - local bits=$(($prefix-$i*8)) - local mask=$((0xff<<$bits & 0xff)) - # TODO: check whether the IP is network/broadcast address - if [ $((${ip_bytes[$i]} & $mask)) -ne ${network_bytes[$i]} ]; then - return 1 - fi - done - - return 0 - fi - - # IPv4 - cidr=${cidr%,*} - local network=${cidr%/*} - local prefix=${cidr#*/} - local ip_hex=$(ipv4_to_hex $ip) - local ip_dec=$((0x$ip_hex)) - local network_hex=$(ipv4_to_hex $network) - local network_dec=$((0x$network_hex)) - local broadcast_dec=$(($network_dec + 2**(32-$prefix) - 1)) - # TODO: check whether the IP is network/broadcast address - if [ $ip_dec -gt $network_dec -a $ip_dec -lt $broadcast_dec ]; then - return 0 - fi - - return 1 -} - -tcpdump(){ - namespacedPod="$1"; shift - namespace=$(echo "$namespacedPod" | cut -d "/" -f1) - podName=$(echo "$namespacedPod" | cut -d "/" -f2) - if [ "$podName" = "$namespacedPod" ]; then - namespace="default" - fi - - nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName}) - hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork}) - - if [ -z "$nodeName" ]; then - echo "Pod $namespacedPod not exists on any node" - exit 1 - fi - - ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -l app=kube-ovn-cni -o 'jsonpath={.items[?(@.spec.nodeName=="'$nodeName'")].metadata.name}') - if [ -z "$ovnCni" ]; then - echo "kube-ovn-cni not exist on node $nodeName" - exit 1 - fi - - if [ "$hostNetwork" = "true" ]; then - set -x - kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- tcpdump -nn "$@" - else - nicName=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r') - if [ -z "$nicName" ]; then - echo "nic doesn't exist on node $nodeName" - exit 1 - fi - podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type}) - podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//') - set -x - if [ "$podNicType" = "internal-port" ]; then - kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" tcpdump -nn -i "$nicName" "$@" - else - kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" tcpdump -nn -i eth0 "$@" - fi - fi -} - -trace(){ - namespacedPod="$1" - namespace=$(echo "$namespacedPod" | cut -d "/" -f1) - podName=$(echo "$namespacedPod" | cut -d "/" -f2) - if [ "$podName" = "$namespacedPod" ]; then - namespace="default" - fi - - dst="$2" - if [ -z "$dst" ]; then - echo "need a target ip address" - exit 1 - fi - - hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork}) - if [ "$hostNetwork" = "true" ]; then - echo "Can not trace host network pod" - exit 1 - fi - - af="4" - nw="nw" - proto="" - if [[ "$dst" =~ .*:.* ]]; then - af="6" - nw="ipv6" - proto="6" - fi - - podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath="{.status.podIPs[*].ip}")) - if [ ${#podIPs[@]} -eq 0 ]; then - podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/ip_address} | sed 's/,/ /g')) - if [ ${#podIPs[@]} -eq 0 ]; then - echo "pod address not ready" - exit 1 - fi - fi - - podIP="" - for ip in ${podIPs[@]}; do - if [ "$af" = "4" ]; then - if [[ ! "$ip" =~ .*:.* ]]; then - podIP=$ip - break - fi - elif [[ "$ip" =~ .*:.* ]]; then - podIP=$ip - break - fi - done - - if [ -z "$podIP" ]; then - echo "Pod $namespacedPod has no IPv$af address" - exit 1 - fi - - nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName}) - ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -l app=kube-ovn-cni -o 'jsonpath={.items[?(@.spec.nodeName=="'$nodeName'")].metadata.name}') - if [ -z "$ovnCni" ]; then - echo "No kube-ovn-cni Pod running on node $nodeName" - exit 1 - fi - - ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch}) - if [ -z "$ls" ]; then - echo "pod address not ready" - exit 1 - fi - - local cidr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/cidr}) - mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address}) - - dstMac="" - if echo "$3" | grep -qE '^([[:xdigit:]]{1,2}:){5}[[:xdigit:]]{1,2}$'; then - dstMac=$3 - shift - elif ipIsInCidr $dst $cidr; then - set +o pipefail - if [ $af -eq 4 ]; then - dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=addresses list logical_switch_port | grep -w "$(echo $dst | tr . '\.')" | awk '{print $1}') - else - dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=addresses list logical_switch_port | grep -i " $dst\$" | awk '{print $1}') - fi - set -o pipefail - fi - - if [ -z "$dstMac" ]; then - vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan}) - logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway}) - if [ ! -z "$vlan" -a "$logicalGateway" != "true" ]; then - gateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.gateway}) - if [[ "$gateway" =~ .*,.* ]]; then - if [ "$af" = "4" ]; then - gateway=${gateway%%,*} - else - gateway=${gateway##*,} - fi - fi - - nicName=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r') - if [ -z "$nicName" ]; then - echo "failed to find ovs interface for Pod namespacedPod on node $nodeName" - exit 1 - fi - - podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type}) - podNetNs=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//') - if [ "$podNicType" != "internal-port" ]; then - interface=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --format=csv --data=bare --no-heading --columns=name find interface external_id:iface-id="$podName"."$namespace") - peer=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ip link show $interface | grep -oE "^[0-9]+:\\s$interface@if[0-9]+" | awk -F @ '{print $2}') - peerIndex=${peer//if/} - peer=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ip link show type veth | grep "^$peerIndex:" | awk -F @ '{print $1}') - nicName=$(echo $peer | awk '{print $2}') - fi - - set +o pipefail - master=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ip link show $nicName | grep -Eo '\smaster\s\w+\s' | awk '{print $2}') - set -o pipefail - if [ ! -z "$master" ]; then - echo "Error: Pod nic $nicName is a slave of $master, please set the destination mac address." - exit 1 - fi - - if [[ "$gateway" =~ .*:.* ]]; then - cmd="ndisc6 -q $gateway $nicName" - output=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ndisc6 -q "$gateway" "$nicName") - else - cmd="arping -c3 -C1 -i1 -I $nicName $gateway" - output=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c3 -C1 -i1 -I "$nicName" "$gateway") - fi - - if [ $? -ne 0 ]; then - echo "Error: failed to execute '$cmd' in Pod's netns" - exit 1 - fi - - dstMac=$(echo "$output" | grep -oE '([[:xdigit:]]{1,2}:){5}[[:xdigit:]]{1,2}') - fi - fi - - if [ -z "$dstMac" ]; then - echo "Using the gateway mac address as destination" - lr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_router}) - if [ -z "$lr" ]; then - lr=$(kubectl get subnet "$ls" -o jsonpath={.spec.vpc}) - fi - dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name="$lr"-"$ls" | tr -d '\r') - fi - - if [ -z "$dstMac" ]; then - echo "failed to get destination mac" - exit 1 - fi - - lsp="$podName.$namespace" - lspUUID=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=_uuid find logical_switch_port name="$lsp") - if [ -z "$lspUUID" ]; then - echo "Notice: LSP $lsp does not exist" - fi - vmOwner=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath='{.metadata.ownerReferences[?(@.kind=="VirtualMachineInstance")].name}') - if [ ! -z "$vmOwner" ]; then - lsp="$vmOwner.$namespace" - fi - - if [ -z "$lsp" ]; then - echo "failed to get LSP of Pod $namespace/$podName" - exit 1 - fi - - type="$3" - case $type in - icmp) - set -x - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$lsp\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $podIP && eth.dst == $dstMac && ip$af.dst == $dst" - ;; - tcp|udp) - set -x - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$lsp\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $podIP && eth.dst == $dstMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4" - ;; - *) - echo "type $type not supported" - echo "kubectl ko trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp or udp port]" - exit 1 - ;; - esac - - set +x - echo "--------" - echo "Start OVS Tracing" - echo "" - echo "" - - inPort=$(kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-vsctl --format=csv --data=bare --no-heading --columns=ofport find interface external_id:iface-id="$podName"."$namespace") - case $type in - icmp) - set -x - kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,icmp$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac" - ;; - tcp|udp) - set -x - kubectl exec "$ovnCni" -c cni-server -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,$type$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac,${type}_src=1000,${type}_dst=$4" - ;; - *) - echo "type $type not supported" - echo "kubectl ko trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp or udp port]" - exit 1 - ;; - esac -} - -xxctl(){ - subcommand="$1"; shift - nodeName="$1"; shift - kubectl get no "$nodeName" > /dev/null - ovsPod=$(kubectl get pod -n $KUBE_OVN_NS -l app=ovs -o 'jsonpath={.items[?(@.spec.nodeName=="'$nodeName'")].metadata.name}') - if [ -z "$ovsPod" ]; then - echo "ovs pod doesn't exist on node $nodeName" - exit 1 - fi - kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-$subcommand "$@" -} - -checkLeader(){ - component="$1"; shift - set +o pipefail - count=$(kubectl get ep ovn-$component -n $KUBE_OVN_NS -o yaml | grep ip | wc -l) - set -o pipefail - if [ $count -eq 0 ]; then - echo "no ovn-$component exists !!" - exit 1 - fi - - if [ $count -gt 1 ]; then - echo "ovn-$component has more than one leader !!" - exit 1 - fi - - echo "ovn-$component leader check ok" -} - -diagnose(){ - kubectl get crd vpcs.kubeovn.io - kubectl get crd vpc-nat-gateways.kubeovn.io - kubectl get crd subnets.kubeovn.io - kubectl get crd ips.kubeovn.io - kubectl get crd vlans.kubeovn.io - kubectl get crd provider-networks.kubeovn.io - set +eu - if ! kubectl get svc kube-dns -n kube-system ; then - echo "Warning: kube-dns doesn't exist, maybe there is coredns service." - fi - set -eu - kubectl get svc kubernetes -n default - kubectl get sa -n kube-system ovn - kubectl get clusterrole system:ovn - kubectl get clusterrolebinding ovn - - kubectl get no -o wide - kubectl ko nbctl show - kubectl ko nbctl lr-policy-list ovn-cluster - kubectl ko nbctl lr-route-list ovn-cluster - kubectl ko nbctl ls-lb-list ovn-default - kubectl ko nbctl list address_set - kubectl ko nbctl list acl - kubectl ko sbctl show - - if [ "${WITHOUT_KUBE_PROXY}" = "false" ]; then - checkKubeProxy - fi - - checkDeployment ovn-central - checkDeployment kube-ovn-controller - checkDaemonSet kube-ovn-cni - checkDaemonSet ovs-ovn - checkDeployment coredns - - checkLeader nb - checkLeader sb - checkLeader northd - - type="$1" - case $type in - all) - echo "### kube-ovn-controller recent log" - set +e - kubectl logs -n $KUBE_OVN_NS -l app=kube-ovn-controller --tail=100 | grep E$(date +%m%d) - set -e - echo "" - pingers=$(kubectl -n $KUBE_OVN_NS get po --no-headers -o custom-columns=NAME:.metadata.name -l app=kube-ovn-pinger) - for pinger in $pingers - do - nodeName=$(kubectl get pod "$pinger" -n "$KUBE_OVN_NS" -o jsonpath={.spec.nodeName}) - echo "### start to diagnose node $nodeName" - echo "#### ovn-controller log:" - kubectl exec -n $KUBE_OVN_NS "$pinger" -- tail /var/log/ovn/ovn-controller.log - echo "" - echo "#### ovs-vswitchd log:" - kubectl exec -n $KUBE_OVN_NS "$pinger" -- tail /var/log/openvswitch/ovs-vswitchd.log - echo "" - echo "#### ovs-vsctl show results:" - kubectl exec -n $KUBE_OVN_NS "$pinger" -- ovs-vsctl show - echo "" - echo "#### pinger diagnose results:" - kubectl exec -n $KUBE_OVN_NS "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job - echo "### finish diagnose node $nodeName" - echo "" - done - ;; - node) - nodeName="$2" - kubectl get no "$nodeName" > /dev/null - pinger=$(kubectl -n $KUBE_OVN_NS get po -l app=kube-ovn-pinger -o 'jsonpath={.items[?(@.spec.nodeName=="'$nodeName'")].metadata.name}') - if [ ! -n "$pinger" ]; then - echo "Error: No kube-ovn-pinger running on node $nodeName" - exit 1 - fi - echo "### start to diagnose node $nodeName" - echo "#### ovn-controller log:" - kubectl exec -n $KUBE_OVN_NS "$pinger" -- tail /var/log/ovn/ovn-controller.log - echo "" - echo "#### ovs-vswitchd log:" - kubectl exec -n $KUBE_OVN_NS "$pinger" -- tail /var/log/openvswitch/ovs-vswitchd.log - echo "" - kubectl exec -n $KUBE_OVN_NS "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job - echo "### finish diagnose node $nodeName" - echo "" - ;; - *) - echo "type $type not supported" - echo "kubectl ko diagnose {all|node} [nodename]" - ;; - esac -} - -getOvnCentralPod(){ - NB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-nb-leader=true | grep ovn-central | head -n 1 | awk '{print $1}') - if [ -z "$NB_POD" ]; then - echo "nb leader not exists" - exit 1 - fi - OVN_NB_POD=$NB_POD - SB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-sb-leader=true | grep ovn-central | head -n 1 | awk '{print $1}') - if [ -z "$SB_POD" ]; then - echo "nb leader not exists" - exit 1 - fi - OVN_SB_POD=$SB_POD - VERSION=$(kubectl -n kube-system get pods -l ovn-sb-leader=true -o yaml | grep "image: $REGISTRY/kube-ovn:" | head -n 1 | awk -F ':' '{print $3}') - if [ -z "$VERSION" ]; then - echo "kubeovn version not exists" - exit 1 - fi - KUBE_OVN_VERSION=$VERSION -} - -checkDaemonSet(){ - name="$1" - currentScheduled=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.currentNumberScheduled}) - desiredScheduled=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.desiredNumberScheduled}) - available=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.numberAvailable}) - ready=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.numberReady}) - if [ "$currentScheduled" = "$desiredScheduled" ] && [ "$desiredScheduled" = "$available" ] && [ "$available" = "$ready" ]; then - echo "ds $name ready" - else - echo "Error ds $name not ready" - exit 1 - fi -} - -checkDeployment(){ - name="$1" - ready=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.readyReplicas}) - updated=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.updatedReplicas}) - desire=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.replicas}) - available=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.availableReplicas}) - if [ "$ready" = "$updated" ] && [ "$updated" = "$desire" ] && [ "$desire" = "$available" ]; then - echo "deployment $name ready" - else - echo "Error deployment $name not ready" - exit 1 - fi -} - -checkKubeProxy(){ - if kubectl get ds -n kube-system --no-headers -o custom-columns=NAME:.metadata.name | grep '^kube-proxy$' >/dev/null; then - checkDaemonSet kube-proxy - else - for node in $(kubectl get node --no-headers -o custom-columns=NAME:.metadata.name); do - local pod=$(kubectl get pod -n $KUBE_OVN_NS -l app=kube-ovn-cni -o 'jsonpath={.items[?(@.spec.nodeName=="'$node'")].metadata.name}') - local ip=$(kubectl get pod -n $KUBE_OVN_NS -l app=kube-ovn-cni -o 'jsonpath={.items[?(@.spec.nodeName=="'$node'")].status.podIP}') - local arg="" - if [[ $ip =~ .*:.* ]]; then - arg="g6" - ip="[$ip]" - fi - healthResult=$(kubectl -n $KUBE_OVN_NS exec $pod -- curl -s${arg} -m 3 -w %{http_code} http://$ip:10256/healthz -o /dev/null | grep -v 200 || true) - if [ -n "$healthResult" ]; then - echo "$node kube-proxy's health check failed" - exit 1 - fi - done - fi - echo "kube-proxy ready" -} - -dbtool(){ - suffix=$(date +%m%d%H%M%s) - component="$1"; shift - action="$1"; shift - case $component in - nb) - case $action in - status) - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl ovsdb-server/get-db-storage-status OVN_Northbound - ;; - kick) - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/kick OVN_Northbound "$1" - ;; - backup) - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnnb_db.$suffix.backup /etc/ovn/ovnnb_db.db - kubectl cp $KUBE_OVN_NS/$OVN_NB_POD:/etc/ovn/ovnnb_db.$suffix.backup $(pwd)/ovnnb_db.$suffix.backup - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnnb_db.$suffix.backup - echo "backup ovn-$component db to $(pwd)/ovnnb_db.$suffix.backup" - ;; - dbstatus) - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-appctl -t /var/run/ovn/ovnnb_db.ctl ovsdb-server/get-db-storage-status OVN_Northbound - ;; - restore) - # set ovn-central replicas to 0 - replicas=$(kubectl get deployment -n $KUBE_OVN_NS ovn-central -o jsonpath={.spec.replicas}) - kubectl scale deployment -n $KUBE_OVN_NS ovn-central --replicas=0 - echo "ovn-central original replicas is $replicas" - - # backup ovn-nb db - declare nodeIpArray - declare podNameArray - declare nodeIps - - if [[ $(kubectl get deployment -n kube-system ovn-central -o jsonpath='{.spec.template.spec.containers[0].env[1]}') =~ "NODE_IPS" ]]; then - nodeIpVals=$(kubectl get deployment -n kube-system ovn-central -o jsonpath='{.spec.template.spec.containers[0].env[1].value}') - nodeIps=(${nodeIpVals//,/ }) - else - nodeIps=$(kubectl get node -lkube-ovn/role=master -o wide | grep -v "INTERNAL-IP" | awk '{print $6}') - fi - firstIP=${nodeIps[0]} - podNames=$(kubectl get pod -n $KUBE_OVN_NS | grep ovs-ovn | awk '{print $1}') - echo "first nodeIP is $firstIP" - - i=0 - for nodeIp in ${nodeIps[@]} - do - for pod in $podNames - do - hostip=$(kubectl get pod -n $KUBE_OVN_NS $pod -o jsonpath={.status.hostIP}) - if [ $nodeIp = $hostip ]; then - nodeIpArray[$i]=$nodeIp - podNameArray[$i]=$pod - i=$(expr $i + 1) - echo "ovs-ovn pod on node $nodeIp is $pod" - break - fi - done - done - - echo "backup nb db file" - kubectl exec -it -n $KUBE_OVN_NS ${podNameArray[0]} -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnnb_db_standalone.db /etc/ovn/ovnnb_db.db - - # mv all db files - for pod in ${podNameArray[@]} - do - kubectl exec -it -n $KUBE_OVN_NS $pod -- mv /etc/ovn/ovnnb_db.db /tmp - kubectl exec -it -n $KUBE_OVN_NS $pod -- mv /etc/ovn/ovnsb_db.db /tmp - done - - # restore db and replicas - echo "restore nb db file, operate in pod ${podNameArray[0]}" - kubectl exec -it -n $KUBE_OVN_NS ${podNameArray[0]} -- mv /etc/ovn/ovnnb_db_standalone.db /etc/ovn/ovnnb_db.db - kubectl scale deployment -n $KUBE_OVN_NS ovn-central --replicas=$replicas - echo "finish restore nb db file and ovn-central replicas" - - echo "recreate ovs-ovn pods" - kubectl delete pod -n $KUBE_OVN_NS -l app=ovs - ;; - *) - echo "unknown action $action" - esac - ;; - sb) - case $action in - status) - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl ovsdb-server/get-db-storage-status OVN_Southbound - ;; - kick) - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/kick OVN_Southbound "$1" - ;; - backup) - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovsdb-tool cluster-to-standalone /etc/ovn/ovnsb_db.$suffix.backup /etc/ovn/ovnsb_db.db - kubectl cp $KUBE_OVN_NS/$OVN_SB_POD:/etc/ovn/ovnsb_db.$suffix.backup $(pwd)/ovnsb_db.$suffix.backup - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- rm -f /etc/ovn/ovnsb_db.$suffix.backup - echo "backup ovn-$component db to $(pwd)/ovnsb_db.$suffix.backup" - ;; - dbstatus) - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-appctl -t /var/run/ovn/ovnsb_db.ctl ovsdb-server/get-db-storage-status OVN_Southbound - ;; - restore) - echo "restore cmd is only used for nb db" - ;; - *) - echo "unknown action $action" - esac - ;; - *) - echo "unknown subcommand $component" - esac -} - -tuning(){ - action="$1"; shift - sys="$1"; shift - case $action in - install-fastpath) - case $sys in - centos7) - docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp/:/tmp/ $REGISTRY/centos7-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh centos install" - while [ ! -f /tmp/kube_ovn_fastpath.ko ]; - do - sleep 1 - done - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - kubectl cp /tmp/kube_ovn_fastpath.ko kube-system/"$i":/tmp/ - done - ;; - centos8) - docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp/:/tmp/ $REGISTRY/centos8-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh centos install" - while [ ! -f /tmp/kube_ovn_fastpath.ko ]; - do - sleep 1 - done - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - kubectl cp /tmp/kube_ovn_fastpath.ko kube-system/"$i":/tmp/ - done - ;; - *) - echo "unknown system $sys" - esac - ;; - local-install-fastpath) - case $sys in - centos7) - # shellcheck disable=SC2145 - docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos7-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh centos local-install $@" - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - kubectl cp /tmp/kube_ovn_fastpath.ko kube-system/"$i":/tmp/ - done - ;; - centos8) - # shellcheck disable=SC2145 - docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos8-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh centos local-install $@" - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - kubectl cp /tmp/kube_ovn_fastpath.ko kube-system/"$i":/tmp/ - done - ;; - *) - echo "unknown system $sys" - esac - ;; - remove-fastpath) - case $sys in - centos) - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - kubectl -n kube-system exec "$i" -- rm -f /tmp/kube_ovn_fastpath.ko - done - ;; - *) - echo "unknown system $sys" - esac - ;; - install-stt) - case $sys in - centos7) - # shellcheck disable=SC2145 - docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos7-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh stt install" - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - for k in /tmp/*.rpm; do - kubectl cp "$k" kube-system/"$i":/tmp/ - done - done - ;; - centos8) - # shellcheck disable=SC2145 - docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos8-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh stt install" - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - for k in /tmp/*.rpm; do - kubectl cp "$k" kube-system/"$i":/tmp/ - done - done - ;; - *) - echo "unknown system $sys" - esac - ;; - local-install-stt) - case $sys in - centos7) - # shellcheck disable=SC2145 - docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos7-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh stt local-install $@" - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - for k in /tmp/*.rpm; do - kubectl cp "$k" kube-system/"$i":/tmp/ - done - done - ;; - centos8) - # shellcheck disable=SC2145 - docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos8-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh stt local-install $@" - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - for k in /tmp/*.rpm; do - kubectl cp "$k" kube-system/"$i":/tmp/ - done - done - ;; - *) - echo "unknown system $sys" - esac - ;; - remove-stt) - case $sys in - centos) - for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}'); - do - kubectl -n kube-system exec "$i" -- rm -f /tmp/openvswitch-kmod*.rpm - done - ;; - *) - echo "unknown system $sys" - esac - ;; - *) - echo "unknown action $action" - esac -} - -reload(){ - kubectl delete pod -n kube-system -l app=ovn-central - kubectl rollout status deployment/ovn-central -n kube-system - kubectl delete pod -n kube-system -l app=ovs - kubectl delete pod -n kube-system -l app=kube-ovn-controller - kubectl rollout status deployment/kube-ovn-controller -n kube-system - kubectl delete pod -n kube-system -l app=kube-ovn-cni - kubectl rollout status daemonset/kube-ovn-cni -n kube-system - kubectl delete pod -n kube-system -l app=kube-ovn-pinger - kubectl rollout status daemonset/kube-ovn-pinger -n kube-system - kubectl delete pod -n kube-system -l app=kube-ovn-monitor - kubectl rollout status deployment/kube-ovn-monitor -n kube-system -} - -env-check(){ - set +e - - KUBE_OVN_NS=kube-system - podNames=$(kubectl get pod --no-headers -n $KUBE_OVN_NS | grep kube-ovn-cni | awk '{print $1}') - for pod in $podNames - do - nodeName=$(kubectl get pod $pod -n $KUBE_OVN_NS -o jsonpath={.spec.nodeName}) - echo "************************************************" - echo "Start environment check for Node $nodeName" - echo "************************************************" - kubectl exec -it -n $KUBE_OVN_NS $pod -c cni-server -- bash /kube-ovn/env-check.sh - done -} - -if [ $# -lt 1 ]; then - showHelp - exit 0 -else - subcommand="$1"; shift -fi - -getOvnCentralPod - -case $subcommand in - nbctl) - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl "$@" - ;; - sbctl) - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-sbctl "$@" - ;; - vsctl|ofctl|dpctl|appctl) - xxctl "$subcommand" "$@" - ;; - nb|sb) - dbtool "$subcommand" "$@" - ;; - tcpdump) - tcpdump "$@" - ;; - trace) - trace "$@" - ;; - diagnose) - diagnose "$@" - ;; - reload) - reload - ;; - tuning) - tuning "$@" - ;; - env-check) - env-check - ;; - *) - showHelp - exit 1 - ;; -esac -`))) diff --git a/cmd/kk/pkg/plugins/network/templates/kubeovn.go b/cmd/kk/pkg/plugins/network/templates/kubeovn.go deleted file mode 100644 index 16fad503..00000000 --- a/cmd/kk/pkg/plugins/network/templates/kubeovn.go +++ /dev/null @@ -1,2780 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var ( - KubeOvnCrd = template.Must(template.New("kube-ovn-crd.yaml").Parse( - dedent.Dedent(`--- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: vpc-nat-gateways.kubeovn.io -spec: - group: kubeovn.io - names: - plural: vpc-nat-gateways - singular: vpc-nat-gateway - shortNames: - - vpc-nat-gw - kind: VpcNatGateway - listKind: VpcNatGatewayList - scope: Cluster - versions: - - additionalPrinterColumns: - - jsonPath: .spec.vpc - name: Vpc - type: string - - jsonPath: .spec.subnet - name: Subnet - type: string - - jsonPath: .spec.lanIp - name: LanIP - type: string - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - lanIp: - type: string - subnet: - type: string - vpc: - type: string - selector: - type: array - items: - type: string ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: iptables-eips.kubeovn.io -spec: - group: kubeovn.io - names: - plural: iptables-eips - singular: iptables-eip - shortNames: - - eip - kind: IptablesEIP - listKind: IptablesEIPList - scope: Cluster - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - additionalPrinterColumns: - - jsonPath: .status.ip - name: IP - type: string - - jsonPath: .spec.macAddress - name: Mac - type: string - - jsonPath: .status.nat - name: Nat - type: string - - jsonPath: .spec.natGwDp - name: NatGwDp - type: string - - jsonPath: .status.ready - name: Ready - type: boolean - schema: - openAPIV3Schema: - type: object - properties: - status: - type: object - properties: - ready: - type: boolean - ip: - type: string - nat: - type: string - redo: - type: string - conditions: - type: array - items: - type: object - properties: - type: - type: string - status: - type: string - reason: - type: string - message: - type: string - lastUpdateTime: - type: string - lastTransitionTime: - type: string - spec: - type: object - properties: - v4ip: - type: string - v6ip: - type: string - macAddress: - type: string - natGwDp: - type: string ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: iptables-fip-rules.kubeovn.io -spec: - group: kubeovn.io - names: - plural: iptables-fip-rules - singular: iptables-fip-rule - shortNames: - - fip - kind: IptablesFIPRule - listKind: IptablesFIPRuleList - scope: Cluster - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - additionalPrinterColumns: - - jsonPath: .spec.eip - name: Eip - type: string - - jsonPath: .status.v4ip - name: V4ip - type: string - - jsonPath: .spec.internalIp - name: InternalIp - type: string - - jsonPath: .status.v6ip - name: V6ip - type: string - - jsonPath: .status.ready - name: Ready - type: boolean - - jsonPath: .status.natGwDp - name: NatGwDp - type: string - schema: - openAPIV3Schema: - type: object - properties: - status: - type: object - properties: - ready: - type: boolean - v4ip: - type: string - v6ip: - type: string - natGwDp: - type: string - redo: - type: string - conditions: - type: array - items: - type: object - properties: - type: - type: string - status: - type: string - reason: - type: string - message: - type: string - lastUpdateTime: - type: string - lastTransitionTime: - type: string - spec: - type: object - properties: - eip: - type: string - internalIp: - type: string ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: iptables-dnat-rules.kubeovn.io -spec: - group: kubeovn.io - names: - plural: iptables-dnat-rules - singular: iptables-dnat-rule - shortNames: - - dnat - kind: IptablesDnatRule - listKind: IptablesDnatRuleList - scope: Cluster - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - additionalPrinterColumns: - - jsonPath: .spec.eip - name: Eip - type: string - - jsonPath: .spec.protocol - name: Protocol - type: string - - jsonPath: .status.v4ip - name: V4ip - type: string - - jsonPath: .status.v6ip - name: V6ip - type: string - - jsonPath: .spec.internalIp - name: InternalIp - type: string - - jsonPath: .spec.externalPort - name: ExternalPort - type: string - - jsonPath: .spec.internalPort - name: InternalPort - type: string - - jsonPath: .status.natGwDp - name: NatGwDp - type: string - - jsonPath: .status.ready - name: Ready - type: boolean - schema: - openAPIV3Schema: - type: object - properties: - status: - type: object - properties: - ready: - type: boolean - v4ip: - type: string - v6ip: - type: string - natGwDp: - type: string - redo: - type: string - conditions: - type: array - items: - type: object - properties: - type: - type: string - status: - type: string - reason: - type: string - message: - type: string - lastUpdateTime: - type: string - lastTransitionTime: - type: string - spec: - type: object - properties: - eip: - type: string - externalPort: - type: string - protocol: - type: string - internalIp: - type: string - internalPort: - type: string ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: iptables-snat-rules.kubeovn.io -spec: - group: kubeovn.io - names: - plural: iptables-snat-rules - singular: iptables-snat-rule - shortNames: - - snat - kind: IptablesSnatRule - listKind: IptablesSnatRuleList - scope: Cluster - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - additionalPrinterColumns: - - jsonPath: .spec.eip - name: EIP - type: string - - jsonPath: .status.v4ip - name: V4ip - type: string - - jsonPath: .status.v6ip - name: V6ip - type: string - - jsonPath: .spec.internalCIDR - name: InternalCIDR - type: string - - jsonPath: .status.natGwDp - name: NatGwDp - type: string - - jsonPath: .status.ready - name: Ready - type: boolean - schema: - openAPIV3Schema: - type: object - properties: - status: - type: object - properties: - ready: - type: boolean - v4ip: - type: string - v6ip: - type: string - natGwDp: - type: string - redo: - type: string - conditions: - type: array - items: - type: object - properties: - type: - type: string - status: - type: string - reason: - type: string - message: - type: string - lastUpdateTime: - type: string - lastTransitionTime: - type: string - spec: - type: object - properties: - eip: - type: string - internalCIDR: - type: string ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: vpcs.kubeovn.io -spec: - group: kubeovn.io - versions: - - additionalPrinterColumns: - - jsonPath: .status.standby - name: Standby - type: boolean - - jsonPath: .status.subnets - name: Subnets - type: string - - jsonPath: .spec.namespaces - name: Namespaces - type: string - name: v1 - schema: - openAPIV3Schema: - properties: - spec: - properties: - namespaces: - items: - type: string - type: array - staticRoutes: - items: - properties: - policy: - type: string - cidr: - type: string - nextHopIP: - type: string - type: object - type: array - policyRoutes: - items: - properties: - priority: - type: integer - action: - type: string - match: - type: string - nextHopIP: - type: string - type: object - type: array - vpcPeerings: - items: - properties: - remoteVpc: - type: string - localConnectIP: - type: string - type: object - type: array - type: object - status: - properties: - conditions: - items: - properties: - lastTransitionTime: - type: string - lastUpdateTime: - type: string - message: - type: string - reason: - type: string - status: - type: string - type: - type: string - type: object - type: array - default: - type: boolean - defaultLogicalSwitch: - type: string - router: - type: string - standby: - type: boolean - subnets: - items: - type: string - type: array - vpcPeerings: - items: - type: string - type: array - tcpLoadBalancer: - type: string - tcpSessionLoadBalancer: - type: string - udpLoadBalancer: - type: string - udpSessionLoadBalancer: - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} - names: - kind: Vpc - listKind: VpcList - plural: vpcs - shortNames: - - vpc - singular: vpc - scope: Cluster ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ips.kubeovn.io -spec: - group: kubeovn.io - versions: - - name: v1 - served: true - storage: true - additionalPrinterColumns: - - name: V4IP - type: string - jsonPath: .spec.v4IpAddress - - name: V6IP - type: string - jsonPath: .spec.v6IpAddress - - name: Mac - type: string - jsonPath: .spec.macAddress - - name: Node - type: string - jsonPath: .spec.nodeName - - name: Subnet - type: string - jsonPath: .spec.subnet - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - podName: - type: string - namespace: - type: string - subnet: - type: string - attachSubnets: - type: array - items: - type: string - nodeName: - type: string - ipAddress: - type: string - v4IpAddress: - type: string - v6IpAddress: - type: string - attachIps: - type: array - items: - type: string - macAddress: - type: string - attachMacs: - type: array - items: - type: string - containerID: - type: string - podType: - type: string - scope: Cluster - names: - plural: ips - singular: ip - kind: IP - shortNames: - - ip ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: vips.kubeovn.io -spec: - group: kubeovn.io - names: - plural: vips - singular: vip - shortNames: - - vip - kind: Vip - listKind: VipList - scope: Cluster - versions: - - name: v1 - served: true - storage: true - additionalPrinterColumns: - - name: V4IP - type: string - jsonPath: .spec.v4ip - - name: PV4IP - type: string - jsonPath: .spec.parentV4ip - - name: Mac - type: string - jsonPath: .spec.macAddress - - name: PMac - type: string - jsonPath: .spec.ParentMac - - name: V6IP - type: string - jsonPath: .spec.v6ip - - name: PV6IP - type: string - jsonPath: .spec.parentV6ip - - name: Subnet - type: string - jsonPath: .spec.subnet - - jsonPath: .status.ready - name: Ready - type: boolean - schema: - openAPIV3Schema: - type: object - properties: - status: - type: object - properties: - ready: - type: boolean - v4ip: - type: string - v6ip: - type: string - mac: - type: string - pv4ip: - type: string - pv6ip: - type: string - pmac: - type: string - conditions: - type: array - items: - type: object - properties: - type: - type: string - status: - type: string - reason: - type: string - message: - type: string - lastUpdateTime: - type: string - lastTransitionTime: - type: string - spec: - type: object - properties: - namespace: - type: string - subnet: - type: string - attachSubnets: - type: array - items: - type: string - v4ip: - type: string - macAddress: - type: string - v6ip: - type: string - parentV4ip: - type: string - parentMac: - type: string - parentV6ip: - type: string ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: subnets.kubeovn.io -spec: - group: kubeovn.io - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - additionalPrinterColumns: - - name: Provider - type: string - jsonPath: .spec.provider - - name: Vpc - type: string - jsonPath: .spec.vpc - - name: Protocol - type: string - jsonPath: .spec.protocol - - name: CIDR - type: string - jsonPath: .spec.cidrBlock - - name: Private - type: boolean - jsonPath: .spec.private - - name: NAT - type: boolean - jsonPath: .spec.natOutgoing - - name: Default - type: boolean - jsonPath: .spec.default - - name: GatewayType - type: string - jsonPath: .spec.gatewayType - - name: V4Used - type: number - jsonPath: .status.v4usingIPs - - name: V4Available - type: number - jsonPath: .status.v4availableIPs - - name: V6Used - type: number - jsonPath: .status.v6usingIPs - - name: V6Available - type: number - jsonPath: .status.v6availableIPs - - name: ExcludeIPs - type: string - jsonPath: .spec.excludeIps - schema: - openAPIV3Schema: - type: object - properties: - status: - type: object - properties: - v4availableIPs: - type: number - v4usingIPs: - type: number - v6availableIPs: - type: number - v6usingIPs: - type: number - activateGateway: - type: string - dhcpV4OptionsUUID: - type: string - dhcpV6OptionsUUID: - type: string - conditions: - type: array - items: - type: object - properties: - type: - type: string - status: - type: string - reason: - type: string - message: - type: string - lastUpdateTime: - type: string - lastTransitionTime: - type: string - spec: - type: object - properties: - vpc: - type: string - default: - type: boolean - protocol: - type: string - enum: - - IPv4 - - IPv6 - - Dual - cidrBlock: - type: string - namespaces: - type: array - items: - type: string - gateway: - type: string - provider: - type: string - excludeIps: - type: array - items: - type: string - vips: - type: array - items: - type: string - gatewayType: - type: string - allowSubnets: - type: array - items: - type: string - gatewayNode: - type: string - natOutgoing: - type: boolean - externalEgressGateway: - type: string - policyRoutingPriority: - type: integer - minimum: 1 - maximum: 32765 - policyRoutingTableID: - type: integer - minimum: 1 - maximum: 2147483647 - not: - enum: - - 252 # compat - - 253 # default - - 254 # main - - 255 # local - private: - type: boolean - vlan: - type: string - logicalGateway: - type: boolean - disableGatewayCheck: - type: boolean - disableInterConnection: - type: boolean - htbqos: - type: string - enableDHCP: - type: boolean - dhcpV4Options: - type: string - dhcpV6Options: - type: string - enableIPv6RA: - type: boolean - ipv6RAConfigs: - type: string - acls: - type: array - items: - type: object - properties: - direction: - type: string - enum: - - from-lport - - to-lport - priority: - type: integer - minimum: 0 - maximum: 32767 - match: - type: string - action: - type: string - enum: - - allow-related - - allow-stateless - - allow - - drop - - reject - scope: Cluster - names: - plural: subnets - singular: subnet - kind: Subnet - shortNames: - - subnet ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: vlans.kubeovn.io -spec: - group: kubeovn.io - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - id: - type: integer - minimum: 0 - maximum: 4095 - provider: - type: string - vlanId: - type: integer - description: Deprecated in favor of id - providerInterfaceName: - type: string - description: Deprecated in favor of provider - required: - - provider - status: - type: object - properties: - subnets: - type: array - items: - type: string - additionalPrinterColumns: - - name: ID - type: string - jsonPath: .spec.id - - name: Provider - type: string - jsonPath: .spec.provider - scope: Cluster - names: - plural: vlans - singular: vlan - kind: Vlan - shortNames: - - vlan ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: provider-networks.kubeovn.io -spec: - group: kubeovn.io - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - schema: - openAPIV3Schema: - type: object - properties: - metadata: - type: object - properties: - name: - type: string - maxLength: 12 - not: - enum: - - int - - external - spec: - type: object - properties: - defaultInterface: - type: string - maxLength: 15 - pattern: '^[^/\s]+$' - customInterfaces: - type: array - items: - type: object - properties: - interface: - type: string - maxLength: 15 - pattern: '^[^/\s]+$' - nodes: - type: array - items: - type: string - exchangeLinkName: - type: boolean - excludeNodes: - type: array - items: - type: string - required: - - defaultInterface - status: - type: object - properties: - ready: - type: boolean - readyNodes: - type: array - items: - type: string - vlans: - type: array - items: - type: string - conditions: - type: array - items: - type: object - properties: - node: - type: string - type: - type: string - status: - type: string - reason: - type: string - message: - type: string - lastUpdateTime: - type: string - lastTransitionTime: - type: string - additionalPrinterColumns: - - name: DefaultInterface - type: string - jsonPath: .spec.defaultInterface - - name: Ready - type: boolean - jsonPath: .status.ready - scope: Cluster - names: - plural: provider-networks - singular: provider-network - kind: ProviderNetwork - listKind: ProviderNetworkList ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: security-groups.kubeovn.io -spec: - group: kubeovn.io - names: - plural: security-groups - singular: security-group - shortNames: - - sg - kind: SecurityGroup - listKind: SecurityGroupList - scope: Cluster - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - ingressRules: - type: array - items: - type: object - properties: - ipVersion: - type: string - protocol: - type: string - priority: - type: integer - remoteType: - type: string - remoteAddress: - type: string - remoteSecurityGroup: - type: string - portRangeMin: - type: integer - portRangeMax: - type: integer - policy: - type: string - egressRules: - type: array - items: - type: object - properties: - ipVersion: - type: string - protocol: - type: string - priority: - type: integer - remoteType: - type: string - remoteAddress: - type: string - remoteSecurityGroup: - type: string - portRangeMin: - type: integer - portRangeMax: - type: integer - policy: - type: string - allowSameGroupTraffic: - type: boolean - status: - type: object - properties: - portGroup: - type: string - allowSameGroupTraffic: - type: boolean - ingressMd5: - type: string - egressMd5: - type: string - ingressLastSyncSuccess: - type: boolean - egressLastSyncSuccess: - type: boolean - subresources: - status: {} - conversion: - strategy: None ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: htbqoses.kubeovn.io -spec: - group: kubeovn.io - versions: - - name: v1 - served: true - storage: true - additionalPrinterColumns: - - name: PRIORITY - type: string - jsonPath: .spec.priority - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - priority: - type: string # Value in range 0 to 4,294,967,295. - scope: Cluster - names: - plural: htbqoses - singular: htbqos - kind: HtbQos - shortNames: - - htbqos -`))) - - OVN = template.Must(template.New("ovn.yaml").Parse( - dedent.Dedent(`--- -{{ if .DpdkMode }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ovn - namespace: kube-system - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.k8s.io/system-only: "true" - name: system:ovn -rules: - - apiGroups: ['policy'] - resources: ['podsecuritypolicies'] - verbs: ['use'] - resourceNames: - - kube-ovn - - apiGroups: - - "kubeovn.io" - resources: - - vpcs - - vpcs/status - - vpc-nat-gateways - - subnets - - subnets/status - - ips - - vips - - vips/status - - vlans - - vlans/status - - provider-networks - - provider-networks/status - - security-groups - - security-groups/status - - htbqoses - - iptables-eips - - iptables-fip-rules - - iptables-dnat-rules - - iptables-snat-rules - - iptables-eips/status - - iptables-fip-rules/status - - iptables-dnat-rules/status - - iptables-snat-rules/status - verbs: - - "*" - - apiGroups: - - "" - resources: - - pods - - pods/exec - - namespaces - - nodes - - configmaps - verbs: - - create - - get - - list - - watch - - patch - - update - - apiGroups: - - "k8s.cni.cncf.io" - resources: - - network-attachment-definitions - verbs: - - create - - delete - - get - - list - - update - - apiGroups: - - "" - - networking.k8s.io - - apps - - extensions - resources: - - networkpolicies - - services - - endpoints - - statefulsets - - daemonsets - - deployments - - deployments/scale - verbs: - - create - - delete - - update - - patch - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - "*" - - apiGroups: - - "k8s.cni.cncf.io" - resources: - - network-attachment-definitions - verbs: - - create - - delete - - get - - list - - update - - apiGroups: - - "kubevirt.io" - resources: - - virtualmachines - - virtualmachineinstances - verbs: - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: ovn -roleRef: - name: system:ovn - kind: ClusterRole - apiGroup: rbac.authorization.k8s.io -subjects: - - kind: ServiceAccount - name: ovn - namespace: kube-system - ---- -kind: Service -apiVersion: v1 -metadata: - name: ovn-nb - namespace: kube-system -spec: - ports: - - name: ovn-nb - protocol: TCP - port: 6641 - targetPort: 6641 - type: ClusterIP - {{ .SvcYamlIpfamilypolicy }} - selector: - app: ovn-central - ovn-nb-leader: "true" - sessionAffinity: None - ---- -kind: Service -apiVersion: v1 -metadata: - name: ovn-sb - namespace: kube-system -spec: - ports: - - name: ovn-sb - protocol: TCP - port: 6642 - targetPort: 6642 - type: ClusterIP - {{ .SvcYamlIpfamilypolicy }} - selector: - app: ovn-central - ovn-sb-leader: "true" - sessionAffinity: None - ---- -kind: Service -apiVersion: v1 -metadata: - name: ovn-northd - namespace: kube-system -spec: - ports: - - name: ovn-northd - protocol: TCP - port: 6643 - targetPort: 6643 - type: ClusterIP - {{ .SvcYamlIpfamilypolicy }} - selector: - app: ovn-central - ovn-northd-leader: "true" - sessionAffinity: None ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: ovn-central - namespace: kube-system - annotations: - kubernetes.io/description: | - OVN components: northd, nb and sb. -spec: - replicas: {{ .Count }} - strategy: - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - type: RollingUpdate - selector: - matchLabels: - app: ovn-central - template: - metadata: - labels: - app: ovn-central - component: network - type: infra - spec: - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: ovn-central - topologyKey: kubernetes.io/hostname - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - containers: - - name: ovn-central - image: "{{ .KubeovnImage }}" - imagePullPolicy: IfNotPresent - command: ["/kube-ovn/start-db.sh"] - securityContext: - capabilities: - add: ["SYS_NICE"] - env: - - name: ENABLE_SSL - value: "{{ .EnableSSL }}" - - name: NODE_IPS - value: {{ .Address }} - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - resources: - requests: - cpu: 300m - memory: 300Mi - limits: - cpu: 3 - memory: 4Gi - volumeMounts: - - mountPath: /var/run/openvswitch - name: host-run-ovs - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /sys - name: host-sys - readOnly: true - - mountPath: /etc/openvswitch - name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /etc/localtime - name: localtime - - mountPath: /var/run/tls - name: kube-ovn-tls - readinessProbe: - exec: - command: - - bash - - /kube-ovn/ovn-healthcheck.sh - periodSeconds: 15 - timeoutSeconds: 45 - livenessProbe: - exec: - command: - - bash - - /kube-ovn/ovn-healthcheck.sh - initialDelaySeconds: 30 - periodSeconds: 15 - failureThreshold: 5 - timeoutSeconds: 45 - nodeSelector: - kubernetes.io/os: "linux" - kube-ovn/role: "master" - volumes: - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: host-sys - hostPath: - path: /sys - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: localtime - hostPath: - path: /etc/localtime - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls - ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: ovs-ovn - namespace: kube-system - annotations: - kubernetes.io/description: | - This daemon set launches the openvswitch daemon. -spec: - selector: - matchLabels: - app: ovs - updateStrategy: - type: OnDelete - template: - metadata: - labels: - app: ovs - component: network - type: infra - spec: - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - hostPID: true - containers: - - name: openvswitch - image: "kubeovn/kube-ovn-dpdk:{{ .DpdkVersion }}-{{ .OvnVersion }}" - imagePullPolicy: IfNotPresent - command: ["/kube-ovn/start-ovs-dpdk.sh"] - securityContext: - runAsUser: 0 - privileged: true - env: - - name: ENABLE_SSL - value: "{{ .EnableSSL }}" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: OVN_DB_IPS - value: {{ .Address }} - volumeMounts: - - mountPath: /var/run/netns - name: host-ns - mountPropagation: HostToContainer - - mountPath: /lib/modules - name: host-modules - readOnly: true - - mountPath: /var/run/openvswitch - name: host-run-ovs - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /sys - name: host-sys - readOnly: true - - mountPath: /etc/cni/net.d - name: cni-conf - - mountPath: /etc/openvswitch - name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /opt/ovs-config - name: host-config-ovs - - mountPath: /dev/hugepages - name: hugepage - - mountPath: /etc/localtime - name: localtime - - mountPath: /var/run/tls - name: kube-ovn-tls - readinessProbe: - exec: - command: - - bash - - /kube-ovn/ovs-dpdk-healthcheck.sh - periodSeconds: 5 - timeoutSeconds: 45 - livenessProbe: - exec: - command: - - bash - - /kube-ovn/ovs-dpdk-healthcheck.sh - initialDelaySeconds: 60 - periodSeconds: 5 - failureThreshold: 5 - timeoutSeconds: 45 - resources: - requests: - cpu: 1000m - memory: 2Gi - limits: - cpu: 1000m - memory: 2Gi - hugepages-1Gi: 1Gi - nodeSelector: - kubernetes.io/os: "linux" - ovn.kubernetes.io/ovs_dp_type: "kernel" - volumes: - - name: host-modules - hostPath: - path: /lib/modules - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: host-sys - hostPath: - path: /sys - - name: host-ns - hostPath: - path: /var/run/netns - - name: cni-conf - hostPath: - path: /etc/cni/net.d - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: host-config-ovs - hostPath: - path: /opt/ovs-config - type: DirectoryOrCreate - - name: hugepage - emptyDir: - medium: HugePages - - name: localtime - hostPath: - path: /etc/localtime - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls -{{ else }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ovn - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.k8s.io/system-only: "true" - name: system:ovn -rules: - - apiGroups: ['policy'] - resources: ['podsecuritypolicies'] - verbs: ['use'] - resourceNames: - - kube-ovn - - apiGroups: - - "kubeovn.io" - resources: - - vpcs - - vpcs/status - - vpc-nat-gateways - - subnets - - subnets/status - - ips - - vips - - vips/status - - vlans - - vlans/status - - provider-networks - - provider-networks/status - - security-groups - - security-groups/status - - htbqoses - - iptables-eips - - iptables-fip-rules - - iptables-dnat-rules - - iptables-snat-rules - - iptables-eips/status - - iptables-fip-rules/status - - iptables-dnat-rules/status - - iptables-snat-rules/status - verbs: - - "*" - - apiGroups: - - "" - resources: - - pods - - pods/exec - - namespaces - - nodes - - configmaps - verbs: - - create - - get - - list - - watch - - patch - - update - - apiGroups: - - "" - - networking.k8s.io - - apps - - extensions - resources: - - networkpolicies - - services - - endpoints - - statefulsets - - daemonsets - - deployments - - deployments/scale - verbs: - - create - - delete - - update - - patch - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - "*" - - apiGroups: - - "k8s.cni.cncf.io" - resources: - - network-attachment-definitions - verbs: - - create - - delete - - get - - list - - update - - apiGroups: - - "kubevirt.io" - resources: - - virtualmachines - - virtualmachineinstances - verbs: - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: ovn -roleRef: - name: system:ovn - kind: ClusterRole - apiGroup: rbac.authorization.k8s.io -subjects: - - kind: ServiceAccount - name: ovn - namespace: kube-system ---- -kind: Service -apiVersion: v1 -metadata: - name: ovn-nb - namespace: kube-system -spec: - ports: - - name: ovn-nb - protocol: TCP - port: 6641 - targetPort: 6641 - type: ClusterIP - {{ .SvcYamlIpfamilypolicy }} - selector: - app: ovn-central - ovn-nb-leader: "true" - sessionAffinity: None ---- -kind: Service -apiVersion: v1 -metadata: - name: ovn-sb - namespace: kube-system -spec: - ports: - - name: ovn-sb - protocol: TCP - port: 6642 - targetPort: 6642 - type: ClusterIP - {{ .SvcYamlIpfamilypolicy }} - selector: - app: ovn-central - ovn-sb-leader: "true" - sessionAffinity: None ---- -kind: Service -apiVersion: v1 -metadata: - name: ovn-northd - namespace: kube-system -spec: - ports: - - name: ovn-northd - protocol: TCP - port: 6643 - targetPort: 6643 - type: ClusterIP - {{ .SvcYamlIpfamilypolicy }} - selector: - app: ovn-central - ovn-northd-leader: "true" - sessionAffinity: None ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: ovn-central - namespace: kube-system - annotations: - kubernetes.io/description: | - OVN components: northd, nb and sb. -spec: - replicas: {{ .Count }} - strategy: - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - type: RollingUpdate - selector: - matchLabels: - app: ovn-central - template: - metadata: - labels: - app: ovn-central - component: network - type: infra - spec: - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: ovn-central - topologyKey: kubernetes.io/hostname - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - containers: - - name: ovn-central - image: "{{ .KubeovnImage }}" - imagePullPolicy: IfNotPresent - command: ["/kube-ovn/start-db.sh"] - securityContext: - capabilities: - add: ["SYS_NICE"] - env: - - name: ENABLE_SSL - value: "{{ .EnableSSL }}" - - name: NODE_IPS - value: {{ .Address }} - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - resources: - requests: - cpu: 300m - memory: 200Mi - limits: - cpu: 3 - memory: 4Gi - volumeMounts: - - mountPath: /var/run/openvswitch - name: host-run-ovs - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /sys - name: host-sys - readOnly: true - - mountPath: /etc/openvswitch - name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /etc/localtime - name: localtime - - mountPath: /var/run/tls - name: kube-ovn-tls - readinessProbe: - exec: - command: - - bash - - /kube-ovn/ovn-healthcheck.sh - periodSeconds: 15 - timeoutSeconds: 45 - livenessProbe: - exec: - command: - - bash - - /kube-ovn/ovn-healthcheck.sh - initialDelaySeconds: 30 - periodSeconds: 15 - failureThreshold: 5 - timeoutSeconds: 45 - nodeSelector: - kubernetes.io/os: "linux" - kube-ovn/role: "master" - volumes: - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: host-sys - hostPath: - path: /sys - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: localtime - hostPath: - path: /etc/localtime - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: ovs-ovn - namespace: kube-system - annotations: - kubernetes.io/description: | - This daemon set launches the openvswitch daemon. -spec: - selector: - matchLabels: - app: ovs - updateStrategy: - type: OnDelete - template: - metadata: - labels: - app: ovs - component: network - type: infra - spec: - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - hostPID: true - containers: - - name: openvswitch - image: "{{ .KubeovnImage }}" - imagePullPolicy: IfNotPresent - command: ["/kube-ovn/start-ovs.sh"] - securityContext: - runAsUser: 0 - privileged: true - env: - - name: ENABLE_SSL - value: "{{ .EnableSSL }}" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: HW_OFFLOAD - value: "{{ .HwOffload }}" - - name: TUNNEL_TYPE - value: "{{ .TunnelType }}" - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: OVN_DB_IPS - value: {{ .Address }} - volumeMounts: - - mountPath: /var/run/netns - name: host-ns - mountPropagation: HostToContainer - - mountPath: /lib/modules - name: host-modules - readOnly: true - - mountPath: /var/run/openvswitch - name: host-run-ovs - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /sys - name: host-sys - readOnly: true - - mountPath: /etc/cni/net.d - name: cni-conf - - mountPath: /etc/openvswitch - name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /etc/localtime - name: localtime - - mountPath: /var/run/tls - name: kube-ovn-tls - readinessProbe: - exec: - command: - - bash - - -c - - LOG_ROTATE=true /kube-ovn/ovs-healthcheck.sh - periodSeconds: 5 - timeoutSeconds: 45 - livenessProbe: - exec: - command: - - bash - - /kube-ovn/ovs-healthcheck.sh - initialDelaySeconds: 60 - periodSeconds: 5 - failureThreshold: 5 - timeoutSeconds: 45 - resources: - requests: - cpu: 200m - memory: 200Mi - limits: - cpu: 1000m - memory: 1000Mi - nodeSelector: - kubernetes.io/os: "linux" - volumes: - - name: host-modules - hostPath: - path: /lib/modules - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: host-sys - hostPath: - path: /sys - - name: host-ns - hostPath: - path: /var/run/netns - - name: cni-conf - hostPath: - path: /etc/cni/net.d - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: localtime - hostPath: - path: /etc/localtime - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls -{{ end }}`))) - - KubeOvn = template.Must(template.New("kube-ovn.yaml").Parse( - dedent.Dedent(`--- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: kube-ovn-controller - namespace: kube-system - annotations: - kubernetes.io/description: | - kube-ovn controller -spec: - replicas: {{ .Count }} - selector: - matchLabels: - app: kube-ovn-controller - strategy: - rollingUpdate: - maxSurge: 0% - maxUnavailable: 100% - type: RollingUpdate - template: - metadata: - labels: - app: kube-ovn-controller - component: network - type: infra - spec: - tolerations: - - effect: NoSchedule - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: kube-ovn-controller - topologyKey: kubernetes.io/hostname - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - containers: - - name: kube-ovn-controller - image: "{{ .KubeovnImage }}" - imagePullPolicy: IfNotPresent - args: - - /kube-ovn/start-controller.sh - - --default-cidr={{ .PodCIDR }} - - --default-gateway={{ .PodGateway }} - - --default-gateway-check={{ .CheckGateway }} - - --default-logical-gateway={{ .LogicalGateway }} - - --default-exclude-ips={{ .ExcludeIps }} - - --node-switch-cidr={{ .JoinCIDR }} - - --service-cluster-ip-range={{ .SvcCIDR }} - - --network-type={{ .NetworkType }} - - --default-interface-name={{ .VlanInterfaceName }} - - --default-vlan-id={{ .VlanID }} - - --pod-nic-type={{ .PodNicType }} - - --enable-lb={{ .EnableLB }} - - --enable-np={{ .EnableNP }} - - --enable-eip-snat={{ .EnableEipSnat }} - - --enable-external-vpc={{ .EnableExternalVPC }} - - --logtostderr=false - - --alsologtostderr=true - - --log_file=/var/log/kube-ovn/kube-ovn-controller.log - - --log_file_max_size=0 - - --keep-vm-ip=true - env: - - name: ENABLE_SSL - value: "{{ .EnableSSL }}" - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KUBE_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: OVN_DB_IPS - value: {{ .Address }} - volumeMounts: - - mountPath: /etc/localtime - name: localtime - - mountPath: /var/log/kube-ovn - name: kube-ovn-log - - mountPath: /var/run/tls - name: kube-ovn-tls - readinessProbe: - exec: - command: - - /kube-ovn/kube-ovn-controller-healthcheck - periodSeconds: 3 - timeoutSeconds: 45 - livenessProbe: - exec: - command: - - /kube-ovn/kube-ovn-controller-healthcheck - initialDelaySeconds: 300 - periodSeconds: 7 - failureThreshold: 5 - timeoutSeconds: 45 - resources: - requests: - cpu: 200m - memory: 200Mi - limits: - cpu: 1000m - memory: 1Gi - nodeSelector: - kubernetes.io/os: "linux" - volumes: - - name: localtime - hostPath: - path: /etc/localtime - - name: kube-ovn-log - hostPath: - path: /var/log/kube-ovn - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls - ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: kube-ovn-cni - namespace: kube-system - annotations: - kubernetes.io/description: | - This daemon set launches the kube-ovn cni daemon. -spec: - selector: - matchLabels: - app: kube-ovn-cni - template: - metadata: - labels: - app: kube-ovn-cni - component: network - type: infra - spec: - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - hostPID: true - initContainers: - - name: install-cni - image: "{{ .KubeovnImage }}" - imagePullPolicy: IfNotPresent - command: ["/kube-ovn/install-cni.sh"] - securityContext: - runAsUser: 0 - privileged: true - volumeMounts: - - mountPath: /opt/cni/bin - name: cni-bin - containers: - - name: cni-server - image: "{{ .KubeovnImage }}" - imagePullPolicy: IfNotPresent - command: - - bash - - /kube-ovn/start-cniserver.sh - args: - - --enable-mirror={{ .EnableMirror }} - - --encap-checksum=true - - --service-cluster-ip-range={{ .SvcCIDR }} - - --iface={{ .Iface }} - - --dpdk-tunnel-iface={{ .DpdkTunnelIface }} - - --network-type={{ .TunnelType }} - - --default-interface-name={{ .VlanInterfaceName }} - - --cni-conf-name={{ .CNIConfigPriority }}-kube-ovn.conflist - - --logtostderr=false - - --alsologtostderr=true - - --log_file=/var/log/kube-ovn/kube-ovn-cni.log - - --log_file_max_size=0 - securityContext: - runAsUser: 0 - privileged: true - env: - - name: ENABLE_SSL - value: "{{ .EnableSSL }}" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: MODULES - value: {{ .Modules }} - - name: RPMS - value: {{ .RPMs }} - volumeMounts: - - name: host-modules - mountPath: /lib/modules - readOnly: true - - name: shared-dir - mountPath: /var/lib/kubelet/pods - - mountPath: /etc/openvswitch - name: systemid - - mountPath: /etc/cni/net.d - name: cni-conf - - mountPath: /run/openvswitch - name: host-run-ovs - mountPropagation: Bidirectional - - mountPath: /run/ovn - name: host-run-ovn - - mountPath: /var/run/netns - name: host-ns - mountPropagation: HostToContainer - - mountPath: /var/log/kube-ovn - name: kube-ovn-log - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /etc/localtime - name: localtime - - mountPath: /tmp - name: tmp - livenessProbe: - failureThreshold: 3 - initialDelaySeconds: 30 - periodSeconds: 7 - successThreshold: 1 - tcpSocket: - port: 10665 - timeoutSeconds: 3 - readinessProbe: - failureThreshold: 3 - initialDelaySeconds: 30 - periodSeconds: 7 - successThreshold: 1 - tcpSocket: - port: 10665 - timeoutSeconds: 3 - resources: - requests: - cpu: 100m - memory: 100Mi - limits: - cpu: 1000m - memory: 1Gi - nodeSelector: - kubernetes.io/os: "linux" - volumes: - - name: host-modules - hostPath: - path: /lib/modules - - name: shared-dir - hostPath: - path: /var/lib/kubelet/pods - - name: systemid - hostPath: - path: /etc/origin/openvswitch - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: cni-conf - hostPath: - path: /etc/cni/net.d - - name: cni-bin - hostPath: - path: /opt/cni/bin - - name: host-ns - hostPath: - path: /var/run/netns - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: kube-ovn-log - hostPath: - path: /var/log/kube-ovn - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: localtime - hostPath: - path: /etc/localtime - - name: tmp - hostPath: - path: /tmp - ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: kube-ovn-pinger - namespace: kube-system - annotations: - kubernetes.io/description: | - This daemon set launches the openvswitch daemon. -spec: - selector: - matchLabels: - app: kube-ovn-pinger - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - app: kube-ovn-pinger - component: network - type: infra - spec: - serviceAccountName: ovn - hostPID: true - containers: - - name: pinger - image: "{{ .KubeovnImage }}" - command: - - /kube-ovn/kube-ovn-pinger - args: - - --external-address={{ .PingExternalAddress }} - - --external-dns={{ .PingExternalDNS }} - - --logtostderr=false - - --alsologtostderr=true - - --log_file=/var/log/kube-ovn/kube-ovn-pinger.log - - --log_file_max_size=0 - imagePullPolicy: IfNotPresent - securityContext: - runAsUser: 0 - privileged: false - env: - - name: ENABLE_SSL - value: "{{ .EnableSSL }}" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - mountPath: /lib/modules - name: host-modules - readOnly: true - - mountPath: /run/openvswitch - name: host-run-ovs - - mountPath: /var/run/openvswitch - name: host-run-ovs - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /sys - name: host-sys - readOnly: true - - mountPath: /etc/openvswitch - name: host-config-openvswitch - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /var/log/kube-ovn - name: kube-ovn-log - - mountPath: /etc/localtime - name: localtime - - mountPath: /var/run/tls - name: kube-ovn-tls - resources: - requests: - cpu: 100m - memory: 100Mi - limits: - cpu: 200m - memory: 400Mi - nodeSelector: - kubernetes.io/os: "linux" - volumes: - - name: host-modules - hostPath: - path: /lib/modules - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: host-sys - hostPath: - path: /sys - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: kube-ovn-log - hostPath: - path: /var/log/kube-ovn - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: localtime - hostPath: - path: /etc/localtime - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: kube-ovn-monitor - namespace: kube-system - annotations: - kubernetes.io/description: | - Metrics for OVN components: northd, nb and sb. -spec: - replicas: 1 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: RollingUpdate - selector: - matchLabels: - app: kube-ovn-monitor - template: - metadata: - labels: - app: kube-ovn-monitor - component: network - type: infra - spec: - tolerations: - - effect: NoSchedule - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: kube-ovn-monitor - topologyKey: kubernetes.io/hostname - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - containers: - - name: kube-ovn-monitor - image: "{{ .KubeovnImage }}" - imagePullPolicy: IfNotPresent - command: ["/kube-ovn/start-ovn-monitor.sh"] - securityContext: - runAsUser: 0 - privileged: false - env: - - name: ENABLE_SSL - value: "{{ .EnableSSL }}" - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - resources: - requests: - cpu: 200m - memory: 200Mi - limits: - cpu: 200m - memory: 200Mi - volumeMounts: - - mountPath: /var/run/openvswitch - name: host-run-ovs - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /etc/openvswitch - name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /etc/localtime - name: localtime - - mountPath: /var/run/tls - name: kube-ovn-tls - readinessProbe: - exec: - command: - - cat - - /var/run/ovn/ovn-controller.pid - periodSeconds: 10 - timeoutSeconds: 45 - livenessProbe: - exec: - command: - - cat - - /var/run/ovn/ovn-controller.pid - initialDelaySeconds: 30 - periodSeconds: 10 - failureThreshold: 5 - timeoutSeconds: 45 - nodeSelector: - kubernetes.io/os: "linux" - kube-ovn/role: "master" - volumes: - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: localtime - hostPath: - path: /etc/localtime - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls ---- -kind: Service -apiVersion: v1 -metadata: - name: kube-ovn-monitor - namespace: kube-system - labels: - app: kube-ovn-monitor -spec: - ports: - - name: metrics - port: 10661 - type: ClusterIP - {{ .SvcYamlIpfamilypolicy }} - selector: - app: kube-ovn-monitor - sessionAffinity: None ---- -kind: Service -apiVersion: v1 -metadata: - name: kube-ovn-pinger - namespace: kube-system - labels: - app: kube-ovn-pinger -spec: - {{ .SvcYamlIpfamilypolicy }} - selector: - app: kube-ovn-pinger - ports: - - port: 8080 - name: metrics ---- -kind: Service -apiVersion: v1 -metadata: - name: kube-ovn-controller - namespace: kube-system - labels: - app: kube-ovn-controller -spec: - {{ .SvcYamlIpfamilypolicy }} - selector: - app: kube-ovn-controller - ports: - - port: 10660 - name: metrics ---- -kind: Service -apiVersion: v1 -metadata: - name: kube-ovn-cni - namespace: kube-system - labels: - app: kube-ovn-cni -spec: - {{ .SvcYamlIpfamilypolicy }} - selector: - app: kube-ovn-cni - ports: - - port: 10665 - name: metrics`))) -) diff --git a/cmd/kk/pkg/plugins/network/templates/multus.go b/cmd/kk/pkg/plugins/network/templates/multus.go deleted file mode 100644 index 4fa47ee7..00000000 --- a/cmd/kk/pkg/plugins/network/templates/multus.go +++ /dev/null @@ -1,229 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var Multus = template.Must(template.New("multus-network-plugin.yaml").Parse( - dedent.Dedent(` ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: network-attachment-definitions.k8s.cni.cncf.io -spec: - group: k8s.cni.cncf.io - scope: Namespaced - names: - plural: network-attachment-definitions - singular: network-attachment-definition - kind: NetworkAttachmentDefinition - shortNames: - - net-attach-def - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing - Working Group to express the intent for attaching pods to one or more logical or physical - networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec' - type: object - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this represen - tation of an object. Servers should convert recognized schemas to the - latest internal value, and may reject unrecognized values. More info: - https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment' - type: object - properties: - config: - description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration' - type: string ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: multus -rules: - - apiGroups: ["k8s.cni.cncf.io"] - resources: - - '*' - verbs: - - '*' - - apiGroups: - - "" - resources: - - pods - - pods/status - verbs: - - get - - update - - apiGroups: - - "" - - events.k8s.io - resources: - - events - verbs: - - create - - patch - - update ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: multus -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: multus -subjects: -- kind: ServiceAccount - name: multus - namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: multus - namespace: kube-system ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: multus-cni-config - namespace: kube-system - labels: - tier: node - app: multus -data: - # NOTE: If you'd prefer to manually apply a configuration file, you may create one here. - # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod - # change the "args" line below from - # - "--multus-conf-file=auto" - # to: - # "--multus-conf-file=/tmp/multus-conf/70-multus.conf" - # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the - # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet. - cni-conf.json: | - { - "name": "multus-cni-network", - "type": "multus", - "capabilities": { - "portMappings": true - }, - "delegates": [ - { - "cniVersion": "0.3.1", - "name": "default-cni-network", - "plugins": [ - { - "type": "flannel", - "name": "flannel.1", - "delegate": { - "isDefaultGateway": true, - "hairpinMode": true - } - }, - { - "type": "portmap", - "capabilities": { - "portMappings": true - } - } - ] - } - ], - "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig" - } ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-multus-ds - namespace: kube-system - labels: - tier: node - app: multus - name: multus -spec: - selector: - matchLabels: - name: multus - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - tier: node - app: multus - name: multus - spec: - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: multus - containers: - - name: kube-multus - image: {{ .MultusImage }} - command: ["/entrypoint.sh"] - args: - - "--multus-conf-file=auto" - - "--cni-version=0.3.1" - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: true - volumeMounts: - - name: cni - mountPath: /host/etc/cni/net.d - - name: cnibin - mountPath: /host/opt/cni/bin - - name: multus-cfg - mountPath: /tmp/multus-conf - terminationGracePeriodSeconds: 10 - volumes: - - name: cni - hostPath: - path: /etc/cni/net.d - - name: cnibin - hostPath: - path: /opt/cni/bin - - name: multus-cfg - configMap: - name: multus-cni-config - items: - - key: cni-conf.json - path: 70-multus.conf -`))) diff --git a/cmd/kk/pkg/plugins/nfd.go b/cmd/kk/pkg/plugins/nfd.go deleted file mode 100644 index 0f53603e..00000000 --- a/cmd/kk/pkg/plugins/nfd.go +++ /dev/null @@ -1,700 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package plugins - -import ( - "path/filepath" - "text/template" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" -) - -// NodeFeatureDiscovery detects hardware features available on each node in a Kubernetes cluster, and advertises those -// features using node labels. - -var ( - NodeFeatureDiscovery = template.Must(template.New("node-feature-discovery.yaml").Parse( - dedent.Dedent(`--- -apiVersion: v1 -kind: Namespace -metadata: - name: node-feature-discovery ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.7.0 - creationTimestamp: null - name: nodefeaturerules.nfd.k8s-sigs.io -spec: - group: nfd.k8s-sigs.io - names: - kind: NodeFeatureRule - listKind: NodeFeatureRuleList - plural: nodefeaturerules - singular: nodefeaturerule - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: NodeFeatureRule resource specifies a configuration for feature-based customization of node objects, such as node labeling. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NodeFeatureRuleSpec describes a NodeFeatureRule. - properties: - rules: - description: Rules is a list of node customization rules. - items: - description: Rule defines a rule for node customization such as labeling. - properties: - labels: - additionalProperties: - type: string - description: Labels to create if the rule matches. - type: object - labelsTemplate: - description: LabelsTemplate specifies a template to expand for dynamically generating multiple labels. Data (after template expansion) must be keys with an optional value ([=]) separated by newlines. - type: string - matchAny: - description: MatchAny specifies a list of matchers one of which must match. - items: - description: MatchAnyElem specifies one sub-matcher of MatchAny. - properties: - matchFeatures: - description: MatchFeatures specifies a set of matcher terms all of which must match. - items: - description: FeatureMatcherTerm defines requirements against one feature set. All requirements (specified as MatchExpressions) are evaluated against each element in the feature set. - properties: - feature: - type: string - matchExpressions: - additionalProperties: - description: "MatchExpression specifies an expression to evaluate against a set of input values. It contains an operator that is applied when matching the input and an array of values that the operator evaluates the input against. \n NB: CreateMatchExpression or MustCreateMatchExpression() should be used for creating new instances. NB: Validate() must be called if Op or Value fields are modified or if a new instance is created from scratch without using the helper functions." - properties: - op: - description: Op is the operator to be applied. - enum: - - In - - NotIn - - InRegexp - - Exists - - DoesNotExist - - Gt - - Lt - - GtLt - - IsTrue - - IsFalse - type: string - value: - description: Value is the list of values that the operand evaluates the input against. Value should be empty if the operator is Exists, DoesNotExist, IsTrue or IsFalse. Value should contain exactly one element if the operator is Gt or Lt and exactly two elements if the operator is GtLt. In other cases Value should contain at least one element. - items: - type: string - type: array - required: - - op - type: object - description: MatchExpressionSet contains a set of MatchExpressions, each of which is evaluated against a set of input values. - type: object - required: - - feature - - matchExpressions - type: object - type: array - required: - - matchFeatures - type: object - type: array - matchFeatures: - description: MatchFeatures specifies a set of matcher terms all of which must match. - items: - description: FeatureMatcherTerm defines requirements against one feature set. All requirements (specified as MatchExpressions) are evaluated against each element in the feature set. - properties: - feature: - type: string - matchExpressions: - additionalProperties: - description: "MatchExpression specifies an expression to evaluate against a set of input values. It contains an operator that is applied when matching the input and an array of values that the operator evaluates the input against. \n NB: CreateMatchExpression or MustCreateMatchExpression() should be used for creating new instances. NB: Validate() must be called if Op or Value fields are modified or if a new instance is created from scratch without using the helper functions." - properties: - op: - description: Op is the operator to be applied. - enum: - - In - - NotIn - - InRegexp - - Exists - - DoesNotExist - - Gt - - Lt - - GtLt - - IsTrue - - IsFalse - type: string - value: - description: Value is the list of values that the operand evaluates the input against. Value should be empty if the operator is Exists, DoesNotExist, IsTrue or IsFalse. Value should contain exactly one element if the operator is Gt or Lt and exactly two elements if the operator is GtLt. In other cases Value should contain at least one element. - items: - type: string - type: array - required: - - op - type: object - description: MatchExpressionSet contains a set of MatchExpressions, each of which is evaluated against a set of input values. - type: object - required: - - feature - - matchExpressions - type: object - type: array - name: - description: Name of the rule. - type: string - vars: - additionalProperties: - type: string - description: Vars is the variables to store if the rule matches. Variables do not directly inflict any changes in the node object. However, they can be referenced from other rules enabling more complex rule hierarchies, without exposing intermediary output values as labels. - type: object - varsTemplate: - description: VarsTemplate specifies a template to expand for dynamically generating multiple variables. Data (after template expansion) must be keys with an optional value ([=]) separated by newlines. - type: string - required: - - name - type: object - type: array - required: - - rules - type: object - required: - - spec - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nfd-master - namespace: node-feature-discovery ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: nfd-master -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - patch - - update - - list -- apiGroups: - - topology.node.k8s.io - resources: - - noderesourcetopologies - verbs: - - create - - get - - update -- apiGroups: - - nfd.k8s-sigs.io - resources: - - nodefeaturerules - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: nfd-master -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nfd-master -subjects: -- kind: ServiceAccount - name: nfd-master - namespace: node-feature-discovery ---- -apiVersion: v1 -data: - nfd-worker.conf: | - #core: - # labelWhiteList: - # noPublish: false - # sleepInterval: 60s - # featureSources: [all] - # labelSources: [all] - # klog: - # addDirHeader: false - # alsologtostderr: false - # logBacktraceAt: - # logtostderr: true - # skipHeaders: false - # stderrthreshold: 2 - # v: 0 - # vmodule: - ## NOTE: the following options are not dynamically run-time configurable - ## and require a nfd-worker restart to take effect after being changed - # logDir: - # logFile: - # logFileMaxSize: 1800 - # skipLogHeaders: false - #sources: - # cpu: - # cpuid: - ## NOTE: whitelist has priority over blacklist - # attributeBlacklist: - # - "BMI1" - # - "BMI2" - # - "CLMUL" - # - "CMOV" - # - "CX16" - # - "ERMS" - # - "F16C" - # - "HTT" - # - "LZCNT" - # - "MMX" - # - "MMXEXT" - # - "NX" - # - "POPCNT" - # - "RDRAND" - # - "RDSEED" - # - "RDTSCP" - # - "SGX" - # - "SSE" - # - "SSE2" - # - "SSE3" - # - "SSE4" - # - "SSE42" - # - "SSSE3" - # attributeWhitelist: - # kernel: - # kconfigFile: "/path/to/kconfig" - # configOpts: - # - "NO_HZ" - # - "X86" - # - "DMI" - # pci: - # deviceClassWhitelist: - # - "0200" - # - "03" - # - "12" - # deviceLabelFields: - # - "class" - # - "vendor" - # - "device" - # - "subsystem_vendor" - # - "subsystem_device" - # usb: - # deviceClassWhitelist: - # - "0e" - # - "ef" - # - "fe" - # - "ff" - # deviceLabelFields: - # - "class" - # - "vendor" - # - "device" - # custom: - # # The following feature demonstrates the capabilities of the matchFeatures - # - name: "my custom rule" - # labels: - # my-ng-feature: "true" - # # matchFeatures implements a logical AND over all matcher terms in the - # # list (i.e. all of the terms, or per-feature matchers, must match) - # matchFeatures: - # - feature: cpu.cpuid - # matchExpressions: - # AVX512F: {op: Exists} - # - feature: cpu.cstate - # matchExpressions: - # enabled: {op: IsTrue} - # - feature: cpu.pstate - # matchExpressions: - # no_turbo: {op: IsFalse} - # scaling_governor: {op: In, value: ["performance"]} - # - feature: cpu.rdt - # matchExpressions: - # RDTL3CA: {op: Exists} - # - feature: cpu.sst - # matchExpressions: - # bf.enabled: {op: IsTrue} - # - feature: cpu.topology - # matchExpressions: - # hardware_multithreading: {op: IsFalse} - # - # - feature: kernel.config - # matchExpressions: - # X86: {op: Exists} - # LSM: {op: InRegexp, value: ["apparmor"]} - # - feature: kernel.loadedmodule - # matchExpressions: - # e1000e: {op: Exists} - # - feature: kernel.selinux - # matchExpressions: - # enabled: {op: IsFalse} - # - feature: kernel.version - # matchExpressions: - # major: {op: In, value: ["5"]} - # minor: {op: Gt, value: ["10"]} - # - # - feature: storage.block - # matchExpressions: - # rotational: {op: In, value: ["0"]} - # dax: {op: In, value: ["0"]} - # - # - feature: network.device - # matchExpressions: - # operstate: {op: In, value: ["up"]} - # speed: {op: Gt, value: ["100"]} - # - # - feature: memory.numa - # matchExpressions: - # node_count: {op: Gt, value: ["2"]} - # - feature: memory.nv - # matchExpressions: - # devtype: {op: In, value: ["nd_dax"]} - # mode: {op: In, value: ["memory"]} - # - # - feature: system.osrelease - # matchExpressions: - # ID: {op: In, value: ["fedora", "centos"]} - # - feature: system.name - # matchExpressions: - # nodename: {op: InRegexp, value: ["^worker-X"]} - # - # - feature: local.label - # matchExpressions: - # custom-feature-knob: {op: Gt, value: ["100"]} - # - # # The following feature demonstrates the capabilities of the matchAny - # - name: "my matchAny rule" - # labels: - # my-ng-feature-2: "my-value" - # # matchAny implements a logical IF over all elements (sub-matchers) in - # # the list (i.e. at least one feature matcher must match) - # matchAny: - # - matchFeatures: - # - feature: kernel.loadedmodule - # matchExpressions: - # driver-module-X: {op: Exists} - # - feature: pci.device - # matchExpressions: - # vendor: {op: In, value: ["8086"]} - # class: {op: In, value: ["0200"]} - # - matchFeatures: - # - feature: kernel.loadedmodule - # matchExpressions: - # driver-module-Y: {op: Exists} - # - feature: usb.device - # matchExpressions: - # vendor: {op: In, value: ["8086"]} - # class: {op: In, value: ["02"]} - # - # # The following features demonstreate label templating capabilities - # - name: "my template rule" - # labelsTemplate: | - # matchFeatures: - # - feature: system.osrelease - # matchExpressions: - # ID: {op: InRegexp, value: ["^open.*"]} - # VERSION_ID.major: {op: In, value: ["13", "15"]} - # - # - name: "my template rule 2" - # matchFeatures: - # - feature: pci.device - # matchExpressions: - # class: {op: InRegexp, value: ["^06"]} - # vendor: ["8086"] - # - feature: cpu.cpuid - # matchExpressions: - # AVX: {op: Exists} - # - # # The following examples demonstrate vars field and back-referencing - # # previous labels and vars - # - name: "my dummy kernel rule" - # labels: - # "my.kernel.feature": "true" - # matchFeatures: - # - feature: kernel.version - # matchExpressions: - # major: {op: Gt, value: ["2"]} - # - # - name: "my dummy rule with no labels" - # vars: - # "my.dummy.var": "1" - # matchFeatures: - # - feature: cpu.cpuid - # matchExpressions: {} - # - # - name: "my rule using backrefs" - # labels: - # "my.backref.feature": "true" - # matchFeatures: - # - feature: rule.matched - # matchExpressions: - # my.kernel.feature: {op: IsTrue} - # my.dummy.var: {op: Gt, value: ["0"]} - # -kind: ConfigMap -metadata: - name: nfd-worker-conf - namespace: node-feature-discovery ---- -apiVersion: v1 -kind: Service -metadata: - name: nfd-master - namespace: node-feature-discovery -spec: - ports: - - port: 8080 - protocol: TCP - selector: - app: nfd-master - type: ClusterIP ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: nfd - name: nfd-master - namespace: node-feature-discovery -spec: - replicas: 1 - selector: - matchLabels: - app: nfd-master - template: - metadata: - labels: - app: nfd-master - spec: - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: node-role.kubernetes.io/master - operator: In - values: - - "" - weight: 1 - - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: In - values: - - "" - weight: 1 - containers: - - args: [] - command: - - nfd-master - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - image: {{ .NFDImage }} - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - /usr/bin/grpc_health_probe - - -addr=:8080 - initialDelaySeconds: 10 - periodSeconds: 10 - name: nfd-master - readinessProbe: - exec: - command: - - /usr/bin/grpc_health_probe - - -addr=:8080 - failureThreshold: 10 - initialDelaySeconds: 5 - periodSeconds: 10 - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsNonRoot: true - volumeMounts: [] - serviceAccount: nfd-master - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Equal - value: "" - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Equal - value: "" - volumes: [] ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app: nfd - name: nfd-worker - namespace: node-feature-discovery -spec: - selector: - matchLabels: - app: nfd-worker - template: - metadata: - labels: - app: nfd-worker - spec: - containers: - - args: - - -server=nfd-master:8080 - command: - - nfd-worker - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - image: {{ .NFDImage }} - imagePullPolicy: IfNotPresent - name: nfd-worker - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsNonRoot: true - volumeMounts: - - mountPath: /host-boot - name: host-boot - readOnly: true - - mountPath: /host-etc/os-release - name: host-os-release - readOnly: true - - mountPath: /host-sys - name: host-sys - readOnly: true - - mountPath: /host-usr/lib - name: host-usr-lib - readOnly: true - - mountPath: /etc/kubernetes/node-feature-discovery/source.d/ - name: source-d - readOnly: true - - mountPath: /etc/kubernetes/node-feature-discovery/features.d/ - name: features-d - readOnly: true - - mountPath: /etc/kubernetes/node-feature-discovery - name: nfd-worker-conf - readOnly: true - dnsPolicy: ClusterFirstWithHostNet - volumes: - - hostPath: - path: /boot - name: host-boot - - hostPath: - path: /etc/os-release - name: host-os-release - - hostPath: - path: /sys - name: host-sys - - hostPath: - path: /usr/lib - name: host-usr-lib - - hostPath: - path: /etc/kubernetes/node-feature-discovery/source.d/ - name: source-d - - hostPath: - path: /etc/kubernetes/node-feature-discovery/features.d/ - name: features-d - - configMap: - name: nfd-worker-conf - name: nfd-worker-conf - - `))) -) - -func DeployNodeFeatureDiscoveryTasks(d *DeployPluginsModule) []task.Interface { - generateNFDManifests := &task.RemoteTask{ - Name: "GenerateNFDManifests", - Desc: "Generate node-feature-discovery manifests", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: &action.Template{ - Template: NodeFeatureDiscovery, - Data: util.Data{ - "NFDImage": images.GetImage(d.Runtime, d.KubeConf, "node-feature-discovery").ImageName(), - }, - Dst: filepath.Join(common.KubeAddonsDir, NodeFeatureDiscovery.Name()), - }, - Parallel: false, - } - - deployNFD := &task.RemoteTask{ - Name: "ApplyNFDManifests", - Desc: "Apply node-feature-discovery manifests", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: new(common.OnlyFirstMaster), - Action: new(ApplyNodeFeatureDiscoveryManifests), - } - - return []task.Interface{ - generateNFDManifests, - deployNFD, - } -} - -type ApplyNodeFeatureDiscoveryManifests struct { - common.KubeAction -} - -func (a *ApplyNodeFeatureDiscoveryManifests) Execute(runtime connector.Runtime) error { - if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/addons/node-feature-discovery.yaml", true); err != nil { - return errors.Wrap(errors.WithStack(err), "apply node-feature-discovery manifests failed") - } - return nil -} diff --git a/cmd/kk/pkg/plugins/storage/modules.go b/cmd/kk/pkg/plugins/storage/modules.go deleted file mode 100644 index c46b4d09..00000000 --- a/cmd/kk/pkg/plugins/storage/modules.go +++ /dev/null @@ -1,80 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package storage - -import ( - "path/filepath" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/storage/templates" -) - -type DeployLocalVolumeModule struct { - common.KubeModule - Skip bool -} - -func (d *DeployLocalVolumeModule) IsSkip() bool { - return d.Skip -} - -func (d *DeployLocalVolumeModule) Init() { - d.Name = "DeployStorageClassModule" - d.Desc = "Deploy cluster storage-class" - - generate := &task.RemoteTask{ - Name: "GenerateOpenEBSManifest", - Desc: "Generate OpenEBS manifest", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(CheckDefaultStorageClass), - }, - Action: &action.Template{ - Template: templates.OpenEBS, - Dst: filepath.Join(common.KubeAddonsDir, templates.OpenEBS.Name()), - Data: util.Data{ - "ProvisionerLocalPVImage": images.GetImage(d.Runtime, d.KubeConf, "provisioner-localpv").ImageName(), - "LinuxUtilsImage": images.GetImage(d.Runtime, d.KubeConf, "linux-utils").ImageName(), - "BasePath": d.KubeConf.Cluster.Storage.OpenEBS.BasePath, - }, - }, - Parallel: true, - } - - deploy := &task.RemoteTask{ - Name: "DeployOpenEBS", - Desc: "Deploy OpenEBS as cluster default StorageClass", - Hosts: d.Runtime.GetHostsByRole(common.Master), - Prepare: &prepare.PrepareCollection{ - new(common.OnlyFirstMaster), - new(CheckDefaultStorageClass), - }, - Action: new(DeployLocalVolume), - Parallel: true, - } - - d.Tasks = []task.Interface{ - generate, - deploy, - } -} diff --git a/cmd/kk/pkg/plugins/storage/prepares.go b/cmd/kk/pkg/plugins/storage/prepares.go deleted file mode 100644 index 7294dfad..00000000 --- a/cmd/kk/pkg/plugins/storage/prepares.go +++ /dev/null @@ -1,48 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package storage - -import ( - "regexp" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -type CheckDefaultStorageClass struct { - common.KubePrepare -} - -func (c *CheckDefaultStorageClass) PreCheck(runtime connector.Runtime) (bool, error) { - output, err := runtime.GetRunner().SudoCmd( - "/usr/local/bin/kubectl get sc --no-headers | grep '(default)' | wc -l", false) - if err != nil { - return false, errors.Wrap(errors.WithStack(err), "check default storageClass failed") - } - - reg := regexp.MustCompile(`([\d])`) - defaultStorageClassNum := reg.FindStringSubmatch(output)[0] - if defaultStorageClassNum == "0" { - return true, nil - } - host := runtime.RemoteHost() - logger.Log.Messagef(host.GetName(), "Default storageClass in cluster is not unique!") - return false, nil -} diff --git a/cmd/kk/pkg/plugins/storage/tasks.go b/cmd/kk/pkg/plugins/storage/tasks.go deleted file mode 100644 index 79da86ae..00000000 --- a/cmd/kk/pkg/plugins/storage/tasks.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package storage - -import ( - "fmt" - "path/filepath" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -type DeployLocalVolume struct { - common.KubeAction -} - -func (d *DeployLocalVolume) Execute(runtime connector.Runtime) error { - cmd := fmt.Sprintf("/usr/local/bin/kubectl apply -f %s", filepath.Join(common.KubeAddonsDir, "local-volume.yaml")) - if _, err := runtime.GetRunner().SudoCmd(cmd, false); err != nil { - return errors.Wrap(errors.WithStack(err), "deploy local-volume.yaml failed") - } - return nil -} diff --git a/cmd/kk/pkg/plugins/storage/templates/openebs.go b/cmd/kk/pkg/plugins/storage/templates/openebs.go deleted file mode 100644 index 2523ecee..00000000 --- a/cmd/kk/pkg/plugins/storage/templates/openebs.go +++ /dev/null @@ -1,177 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -// OpenEBS defines the template of openebs' manifests. -var OpenEBS = template.Must(template.New("local-volume.yaml").Parse( - dedent.Dedent(`--- -#Sample storage classes for OpenEBS Local PV -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: local - annotations: - storageclass.kubesphere.io/supported-access-modes: '["ReadWriteOnce"]' - storageclass.beta.kubernetes.io/is-default-class: "true" - openebs.io/cas-type: local - cas.openebs.io/config: | - - name: StorageType - value: "hostpath" - - name: BasePath - value: "{{ .BasePath }}" -provisioner: openebs.io/local -volumeBindingMode: WaitForFirstConsumer -reclaimPolicy: Delete ---- -# Create Maya Service Account -apiVersion: v1 -kind: ServiceAccount -metadata: - name: openebs-maya-operator - namespace: kube-system ---- -# Define Role that allows operations on K8s pods/deployments -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: openebs-maya-operator -rules: -- apiGroups: ["*"] - resources: ["nodes", "nodes/proxy"] - verbs: ["*"] -- apiGroups: ["*"] - resources: ["namespaces", "services", "pods", "pods/exec", "deployments", "deployments/finalizers", "replicationcontrollers", "replicasets", "events", "endpoints", "configmaps", "secrets", "jobs", "cronjobs"] - verbs: ["*"] -- apiGroups: ["*"] - resources: ["statefulsets", "daemonsets"] - verbs: ["*"] -- apiGroups: ["*"] - resources: ["resourcequotas", "limitranges"] - verbs: ["list", "watch"] -- apiGroups: ["*"] - resources: ["ingresses", "horizontalpodautoscalers", "verticalpodautoscalers", "poddisruptionbudgets", "certificatesigningrequests"] - verbs: ["list", "watch"] -- apiGroups: ["*"] - resources: ["storageclasses", "persistentvolumeclaims", "persistentvolumes"] - verbs: ["*"] -- apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: [ "get", "list", "create", "update", "delete", "patch"] -- apiGroups: ["openebs.io"] - resources: [ "*"] - verbs: ["*"] -- nonResourceURLs: ["/metrics"] - verbs: ["get"] ---- -# Bind the Service Account with the Role Privileges. -# TODO: Check if default account also needs to be there -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: openebs-maya-operator -subjects: -- kind: ServiceAccount - name: openebs-maya-operator - namespace: kube-system -roleRef: - kind: ClusterRole - name: openebs-maya-operator - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: openebs-localpv-provisioner - namespace: kube-system - labels: - name: openebs-localpv-provisioner - openebs.io/component-name: openebs-localpv-provisioner - openebs.io/version: 3.3.0 -spec: - selector: - matchLabels: - name: openebs-localpv-provisioner - openebs.io/component-name: openebs-localpv-provisioner - replicas: 1 - strategy: - type: Recreate - template: - metadata: - labels: - name: openebs-localpv-provisioner - openebs.io/component-name: openebs-localpv-provisioner - openebs.io/version: 3.3.0 - spec: - serviceAccountName: openebs-maya-operator - containers: - - name: openebs-provisioner-hostpath - imagePullPolicy: IfNotPresent - image: {{ .ProvisionerLocalPVImage }} - env: - # OPENEBS_IO_K8S_MASTER enables openebs provisioner to connect to K8s - # based on this address. This is ignored if empty. - # This is supported for openebs provisioner version 0.5.2 onwards - #- name: OPENEBS_IO_K8S_MASTER - # value: "http://10.128.0.12:8080" - # OPENEBS_IO_KUBE_CONFIG enables openebs provisioner to connect to K8s - # based on this config. This is ignored if empty. - # This is supported for openebs provisioner version 0.5.2 onwards - #- name: OPENEBS_IO_KUBE_CONFIG - # value: "/home/ubuntu/.kube/config" - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: OPENEBS_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - # OPENEBS_SERVICE_ACCOUNT provides the service account of this pod as - # environment variable - - name: OPENEBS_SERVICE_ACCOUNT - valueFrom: - fieldRef: - fieldPath: spec.serviceAccountName - - name: OPENEBS_IO_ENABLE_ANALYTICS - value: "true" - - name: OPENEBS_IO_INSTALLER_TYPE - value: "openebs-operator-lite" - - name: OPENEBS_IO_HELPER_IMAGE - value: "{{ .LinuxUtilsImage }}" - # LEADER_ELECTION_ENABLED is used to enable/disable leader election. By default - # leader election is enabled. - #- name: LEADER_ELECTION_ENABLED - # value: "true" - # OPENEBS_IO_IMAGE_PULL_SECRETS environment variable is used to pass the image pull secrets - # to the helper pod launched by local-pv hostpath provisioner - #- name: OPENEBS_IO_IMAGE_PULL_SECRETS - # value: "" - livenessProbe: - exec: - command: - - sh - - -c - - test $(pgrep -c "^provisioner-loc.*") = 1 - initialDelaySeconds: 30 - periodSeconds: 60 - `))) diff --git a/cmd/kk/pkg/registry/docker_registry_config.go b/cmd/kk/pkg/registry/docker_registry_config.go deleted file mode 100644 index d8d79b6d..00000000 --- a/cmd/kk/pkg/registry/docker_registry_config.go +++ /dev/null @@ -1,123 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package registry - -import ( - "encoding/json" - "os" - "path/filepath" - "strings" - - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/runtime" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" -) - -type DockerRegistryEntry struct { - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - SkipTLSVerify bool `json:"skipTLSVerify,omitempty"` - PlainHTTP bool `json:"plainHTTP,omitempty"` - CertsPath string `json:"certsPath,omitempty"` - // CAFile is an SSL Certificate Authority file used to secure etcd communication. - CAFile string `yaml:"caFile" json:"caFile,omitempty"` - // CertFile is an SSL certification file used to secure etcd communication. - CertFile string `yaml:"certFile" json:"certFile,omitempty"` - // KeyFile is an SSL key file used to secure etcd communication. - KeyFile string `yaml:"keyFile" json:"keyFile,omitempty"` -} - -func DockerRegistryAuthEntries(auths runtime.RawExtension) (entries map[string]*DockerRegistryEntry) { - - if len(auths.Raw) == 0 { - return - } - - err := json.Unmarshal(auths.Raw, &entries) - if err != nil { - logger.Log.Fatalf("Failed to Parse Registry Auths configuration: %v", auths.Raw) - return - } - - for _, v := range entries { - if v.CertsPath != "" { - ca, cert, key, err := LookupCertsFile(v.CertsPath) - if err != nil { - logger.Log.Warningf("Failed to lookup certs file from the specific cert path %s: %s", v.CertsPath, err.Error()) - return - } - v.CAFile = ca - v.CertFile = cert - v.KeyFile = key - } - if v.PlainHTTP { - v.SkipTLSVerify = true - } - } - - return -} - -func LookupCertsFile(path string) (ca string, cert string, key string, err error) { - logger.Log.Debugf("Looking for TLS certificates and private keys in %s", path) - absPath, err := filepath.Abs(path) - if err != nil { - return - } - logger.Log.Debugf("Looking for TLS certificates and private keys in abs path %s", absPath) - entries, err := os.ReadDir(absPath) - if err != nil { - return ca, cert, key, err - } - - for _, f := range entries { - fullPath := filepath.Join(path, f.Name()) - if strings.HasSuffix(f.Name(), ".crt") { - logger.Log.Debugf(" crt: %s", fullPath) - ca = fullPath - } - if strings.HasSuffix(f.Name(), ".cert") { - certName := f.Name() - keyName := certName[:len(certName)-5] + ".key" - logger.Log.Debugf(" cert: %s", fullPath) - if !hasFile(entries, keyName) { - return ca, cert, key, errors.Errorf("missing key %s for client certificate %s. Note that CA certificates should use the extension .crt", keyName, certName) - } - cert = fullPath - } - if strings.HasSuffix(f.Name(), ".key") { - keyName := f.Name() - certName := keyName[:len(keyName)-4] + ".cert" - logger.Log.Debugf(" key: %s", fullPath) - if !hasFile(entries, certName) { - return ca, cert, key, errors.Errorf("missing client certificate %s for key %s", certName, keyName) - } - key = fullPath - } - } - return ca, cert, key, nil -} - -func hasFile(files []os.DirEntry, name string) bool { - for _, f := range files { - if f.Name() == name { - return true - } - } - return false -} diff --git a/cmd/kk/pkg/registry/k3s_registries.go b/cmd/kk/pkg/registry/k3s_registries.go deleted file mode 100644 index aed02883..00000000 --- a/cmd/kk/pkg/registry/k3s_registries.go +++ /dev/null @@ -1,60 +0,0 @@ -package registry - -// Mirror contains the config related to the registry mirror -type Mirror struct { - // Endpoints are endpoints for a namespace. CRI plugin will try the endpoints - // one by one until a working one is found. The endpoint must be a valid url - // with host specified. - // The scheme, host and path from the endpoint URL will be used. - Endpoints []string `toml:"endpoint" yaml:"endpoint"` - - // Rewrites are repository rewrite rules for a namespace. When fetching image resources - // from an endpoint and a key matches the repository via regular expression matching - // it will be replaced with the corresponding value from the map in the resource request. - Rewrites map[string]string `toml:"rewrite" yaml:"rewrite"` -} - -// AuthConfig contains the config related to authentication to a specific registry -type AuthConfig struct { - // Username is the username to login the registry. - Username string `toml:"username" yaml:"username"` - // Password is the password to login the registry. - Password string `toml:"password" yaml:"password"` - // Auth is a base64 encoded string from the concatenation of the username, - // a colon, and the password. - Auth string `toml:"auth" yaml:"auth"` - // IdentityToken is used to authenticate the user and get - // an access token for the registry. - IdentityToken string `toml:"identitytoken" yaml:"identity_token"` -} - -// TLSConfig contains the CA/Cert/Key used for a registry -type TLSConfig struct { - CAFile string `toml:"ca_file" yaml:"ca_file"` - CertFile string `toml:"cert_file" yaml:"cert_file"` - KeyFile string `toml:"key_file" yaml:"key_file"` - InsecureSkipVerify bool `toml:"insecure_skip_verify" yaml:"insecure_skip_verify"` -} - -// Registry is registry settings including mirrors, TLS, and credentials -type Registry struct { - // Mirrors are namespace to mirror mapping for all namespaces. - Mirrors map[string]Mirror `toml:"mirrors" yaml:"mirrors"` - // Configs are configs for each registry. - // The key is the FDQN or IP of the registry. - Configs map[string]RegistryConfig `toml:"configs" yaml:"configs"` - - // Auths are registry endpoint to auth config mapping. The registry endpoint must - // be a valid url with host specified. - // DEPRECATED: Use Configs instead. Remove in containerd 1.4. - Auths map[string]AuthConfig `toml:"auths" yaml:"auths"` -} - -// RegistryConfig contains configuration used to communicate with the registry. -type RegistryConfig struct { - // Auth contains information to authenticate to the registry. - Auth *AuthConfig `toml:"auth" yaml:"auth"` - // TLS is a pair of CA/Cert/Key which then are used when creating the transport - // that communicates with the registry. - TLS *TLSConfig `toml:"tls" yaml:"tls"` -} diff --git a/cmd/kk/pkg/utils/certs/certs.go b/cmd/kk/pkg/utils/certs/certs.go deleted file mode 100644 index e66ea57b..00000000 --- a/cmd/kk/pkg/utils/certs/certs.go +++ /dev/null @@ -1,333 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package certs - -import ( - "crypto" - cryptorand "crypto/rand" - "crypto/x509" - "crypto/x509/pkix" - "fmt" - "math" - "math/big" - "net" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/util/sets" - certutil "k8s.io/client-go/util/cert" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" -) - -const ( - // CertificateValidity defines the validity for all the signed certificates generated by kubeadm - CertificateValidity = time.Hour * 24 * 365 * 10 - // CertificateBlockType is a possible value for pem.Block.Type. - CertificateBlockType = "CERTIFICATE" - rsaKeySize = 2048 -) - -// KubekeyCert represents a certificate that Kubeadm will create to function properly. -type KubekeyCert struct { - Name string - LongName string - BaseName string - CAName string - Config CertConfig -} - -// GetConfig returns the definition for the given cert given the provided InitConfiguration -func (k *KubekeyCert) GetConfig(_ *common.KubeConf) (*CertConfig, error) { - - return &k.Config, nil -} - -// CreateFromCA makes and writes a certificate using the given CA cert and key. -func (k *KubekeyCert) CreateFromCA(kubeConf *common.KubeConf, pkiPath string, caCert *x509.Certificate, caKey crypto.Signer) error { - cfg, err := k.GetConfig(kubeConf) - if err != nil { - return errors.Wrapf(err, "couldn't create %q certificate", k.Name) - } - cert, key, err := NewCertAndKey(caCert, caKey, cfg) - if err != nil { - return err - } - err = writeCertificateFilesIfNotExist( - pkiPath, - k.BaseName, - caCert, - cert, - key, - cfg, - ) - - if err != nil { - return errors.Wrapf(err, "failed to write or validate certificate %q", k.Name) - } - - return nil -} - -func GenerateCA(ca *KubekeyCert, pkiPath string, kubeConf *common.KubeConf) error { - - if cert, err := TryLoadCertFromDisk(pkiPath, ca.BaseName); err == nil { - CheckCertificatePeriodValidity(ca.BaseName, cert) - - if _, err := TryLoadKeyFromDisk(pkiPath, ca.BaseName); err == nil { - fmt.Printf("[certs] Using existing %s certificate authority\n", ca.BaseName) - return nil - } - fmt.Printf("[certs] Using existing %s keyless certificate authority\n", ca.BaseName) - return nil - } - - // create the new certificate authority (or use existing) - return CreateCACertAndKeyFiles(ca, pkiPath, kubeConf) - -} - -// CreateCACertAndKeyFiles generates and writes out a given certificate authority. -// The certSpec should be one of the variables from this package. -func CreateCACertAndKeyFiles(certSpec *KubekeyCert, pkiPath string, kubeConf *common.KubeConf) error { - if certSpec.CAName != "" { - return errors.Errorf("this function should only be used for CAs, but cert %s has CA %s", certSpec.Name, certSpec.CAName) - } - - certConfig, err := certSpec.GetConfig(kubeConf) - if err != nil { - return err - } - - caCert, caKey, err := NewCertificateAuthority(certConfig) - if err != nil { - return err - } - - return writeCertificateAuthorityFilesIfNotExist( - pkiPath, - certSpec.BaseName, - caCert, - caKey, - ) -} - -func GenerateCerts(cert *KubekeyCert, caCert *KubekeyCert, pkiPath string, kubeConf *common.KubeConf) error { - // TODO: if using external etcd, skips etcd certificates generation - - if certData, intermediates, err := TryLoadCertChainFromDisk(pkiPath, cert.BaseName); err == nil { - CheckCertificatePeriodValidity(cert.BaseName, certData) - - caCertData, err := TryLoadCertFromDisk(pkiPath, caCert.BaseName) - if err != nil { - return errors.Wrapf(err, "couldn't load CA certificate %s", caCert.Name) - } - - CheckCertificatePeriodValidity(caCert.BaseName, caCertData) - - if err := VerifyCertChain(certData, intermediates, caCertData); err != nil { - return errors.Wrapf(err, "[certs] certificate %s not signed by CA certificate %s", cert.BaseName, caCert.BaseName) - } - - fmt.Printf("[certs] Using existing %s certificate and key on disk\n", cert.BaseName) - return nil - } - - // create the new certificate (or use existing) - return CreateCertAndKeyFilesWithCA(caCert, cert, pkiPath, kubeConf) -} - -// CreateCertAndKeyFilesWithCA loads the given certificate authority from disk, then generates and writes out the given certificate and key. -// The certSpec and caCertSpec should both be one of the variables from this package. -func CreateCertAndKeyFilesWithCA(caCertSpec *KubekeyCert, certSpec *KubekeyCert, pkiPath string, kubeConf *common.KubeConf) error { - if certSpec.CAName != caCertSpec.Name { - return errors.Errorf("expected CAname for %s to be %q, but was %s", certSpec.Name, certSpec.CAName, caCertSpec.Name) - } - - caCert, caKey, err := LoadCertificateAuthority(pkiPath, caCertSpec.BaseName) - if err != nil { - return errors.Wrapf(err, "couldn't load CA certificate %s", caCertSpec.Name) - } - - return certSpec.CreateFromCA(kubeConf, pkiPath, caCert, caKey) -} - -// LoadCertificateAuthority tries to load a CA in the given directory with the given name. -func LoadCertificateAuthority(pkiDir string, baseName string) (*x509.Certificate, crypto.Signer, error) { - // Checks if certificate authority exists in the PKI directory - if !CertOrKeyExist(pkiDir, baseName) { - return nil, nil, errors.Errorf("couldn't load %s certificate authority from %s", baseName, pkiDir) - } - - // Try to load certificate authority .crt and .key from the PKI directory - caCert, caKey, err := TryLoadCertAndKeyFromDisk(pkiDir, baseName) - if err != nil { - return nil, nil, errors.Wrapf(err, "failure loading %s certificate authority", baseName) - } - // Validate period - CheckCertificatePeriodValidity(baseName, caCert) - - // Make sure the loaded CA cert actually is a CA - if !caCert.IsCA { - return nil, nil, errors.Errorf("%s certificate is not a certificate authority", baseName) - } - - return caCert, caKey, nil -} - -// NewCertAndKey creates new certificate and key by passing the certificate authority certificate and key -func NewCertAndKey(caCert *x509.Certificate, caKey crypto.Signer, config *CertConfig) (*x509.Certificate, crypto.Signer, error) { - if len(config.Usages) == 0 { - return nil, nil, errors.New("must specify at least one ExtKeyUsage") - } - - key, err := NewPrivateKey(config.PublicKeyAlgorithm) - if err != nil { - return nil, nil, errors.Wrap(err, "unable to create private key") - } - - cert, err := NewSignedCert(config, key, caCert, caKey, false) - if err != nil { - return nil, nil, errors.Wrap(err, "unable to sign certificate") - } - - return cert, key, nil -} - -// NewSignedCert creates a signed certificate using the given CA certificate and key -func NewSignedCert(cfg *CertConfig, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer, isCA bool) (*x509.Certificate, error) { - serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64)) - if err != nil { - return nil, err - } - if len(cfg.CommonName) == 0 { - return nil, errors.New("must specify a CommonName") - } - - keyUsage := x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature - if isCA { - keyUsage |= x509.KeyUsageCertSign - } - - RemoveDuplicateAltNames(&cfg.AltNames) - - notAfter := time.Now().Add(CertificateValidity).UTC() - if cfg.NotAfter != nil { - notAfter = *cfg.NotAfter - } - - certTmpl := x509.Certificate{ - Subject: pkix.Name{ - CommonName: cfg.CommonName, - Organization: cfg.Organization, - }, - DNSNames: cfg.AltNames.DNSNames, - IPAddresses: cfg.AltNames.IPs, - SerialNumber: serial, - NotBefore: caCert.NotBefore, - NotAfter: notAfter, - KeyUsage: keyUsage, - ExtKeyUsage: cfg.Usages, - BasicConstraintsValid: true, - IsCA: isCA, - } - certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey) - if err != nil { - return nil, err - } - return x509.ParseCertificate(certDERBytes) -} - -// RemoveDuplicateAltNames removes duplicate items in altNames. -func RemoveDuplicateAltNames(altNames *certutil.AltNames) { - if altNames == nil { - return - } - - if altNames.DNSNames != nil { - altNames.DNSNames = sets.NewString(altNames.DNSNames...).List() - } - - ipsKeys := make(map[string]struct{}) - var ips []net.IP - for _, one := range altNames.IPs { - if _, ok := ipsKeys[one.String()]; !ok { - ipsKeys[one.String()] = struct{}{} - ips = append(ips, one) - } - } - altNames.IPs = ips -} - -// CheckCertificatePeriodValidity takes a certificate and prints a warning if its period -// is not valid related to the current time. It does so only if the certificate was not validated already -// by keeping track with a cache. -func CheckCertificatePeriodValidity(baseName string, cert *x509.Certificate) { - certPeriodValidationMutex.Lock() - defer certPeriodValidationMutex.Unlock() - if _, exists := certPeriodValidation[baseName]; exists { - return - } - certPeriodValidation[baseName] = struct{}{} - - if err := ValidateCertPeriod(cert, 0); err != nil { - logrus.Warningf("WARNING: could not validate bounds for certificate %s: %v", baseName, err) - } -} - -// writeCertificateFilesIfNotExist write a new certificate to the given path. -// If there already is a certificate file at the given path; kubeadm tries to load it and check if the values in the -// existing and the expected certificate equals. If they do; kubeadm will just skip writing the file as it's up-to-date, -// otherwise this function returns an error. -func writeCertificateFilesIfNotExist(pkiDir string, baseName string, signingCert *x509.Certificate, cert *x509.Certificate, key crypto.Signer, cfg *CertConfig) error { - - // Checks if the signed certificate exists in the PKI directory - if CertOrKeyExist(pkiDir, baseName) { - // Try to load key from the PKI directory - _, err := TryLoadKeyFromDisk(pkiDir, baseName) - if err != nil { - return errors.Wrapf(err, "failure loading %s key", baseName) - } - - // Try to load certificate from the PKI directory - signedCert, intermediates, err := TryLoadCertChainFromDisk(pkiDir, baseName) - if err != nil { - return errors.Wrapf(err, "failure loading %s certificate", baseName) - } - // Validate period - CheckCertificatePeriodValidity(baseName, signedCert) - - // Check if the existing cert is signed by the given CA - if err := VerifyCertChain(signedCert, intermediates, signingCert); err != nil { - return errors.Errorf("certificate %s is not signed by corresponding CA", baseName) - } - - // Check if the certificate has the correct attributes - if err := validateCertificateWithConfig(signedCert, baseName, cfg); err != nil { - return err - } - - fmt.Printf("[certs] Using the existing %q certificate and key\n", baseName) - } else { - if err := WriteCertAndKey(pkiDir, baseName, cert, key); err != nil { - return errors.Wrapf(err, "failure while saving %s certificate and key", baseName) - } - if HasServerAuth(cert) { - fmt.Printf("[certs] %s serving cert is signed for DNS names %v and IPs %v\n", baseName, cert.DNSNames, cert.IPAddresses) - } - } - - return nil -} diff --git a/cmd/kk/pkg/utils/certs/utils.go b/cmd/kk/pkg/utils/certs/utils.go deleted file mode 100644 index ead7b68b..00000000 --- a/cmd/kk/pkg/utils/certs/utils.go +++ /dev/null @@ -1,321 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package certs - -import ( - "crypto" - "crypto/ecdsa" - "crypto/elliptic" - cryptorand "crypto/rand" - "crypto/rsa" - "crypto/x509" - "encoding/pem" - "fmt" - "os" - "path/filepath" - "sync" - "time" - - "github.com/pkg/errors" - certutil "k8s.io/client-go/util/cert" - "k8s.io/client-go/util/keyutil" -) - -// CertOrKeyExist returns a boolean whether the cert or the key exists -func CertOrKeyExist(pkiPath, name string) bool { - certificatePath, privateKeyPath := PathsForCertAndKey(pkiPath, name) - - _, certErr := os.Stat(certificatePath) - _, keyErr := os.Stat(privateKeyPath) - if os.IsNotExist(certErr) && os.IsNotExist(keyErr) { - // The cert and the key do not exist - return false - } - - // Both files exist or one of them - return true -} - -// PathsForCertAndKey returns the paths for the certificate and key given the path and basename. -func PathsForCertAndKey(pkiPath, name string) (string, string) { - return pathForCert(pkiPath, name), pathForKey(pkiPath, name) -} - -func pathForCert(pkiPath, name string) string { - return filepath.Join(pkiPath, fmt.Sprintf("%s.pem", name)) -} - -func pathForKey(pkiPath, name string) string { - return filepath.Join(pkiPath, fmt.Sprintf("%s-key.pem", name)) -} - -// TryLoadKeyFromDisk tries to load the key from the disk and validates that it is valid -func TryLoadKeyFromDisk(pkiPath, name string) (crypto.Signer, error) { - privateKeyPath := pathForKey(pkiPath, name) - - // Parse the private key from a file - privKey, err := keyutil.PrivateKeyFromFile(privateKeyPath) - if err != nil { - return nil, errors.Wrapf(err, "couldn't load the private key file %s", privateKeyPath) - } - - // Allow RSA and ECDSA formats only - var key crypto.Signer - switch k := privKey.(type) { - case *rsa.PrivateKey: - key = k - case *ecdsa.PrivateKey: - key = k - default: - return nil, errors.Errorf("the private key file %s is neither in RSA nor ECDSA format", privateKeyPath) - } - - return key, nil -} - -// TryLoadCertChainFromDisk tries to load the cert chain from the disk -func TryLoadCertChainFromDisk(pkiPath, name string) (*x509.Certificate, []*x509.Certificate, error) { - certificatePath := pathForCert(pkiPath, name) - - certs, err := certutil.CertsFromFile(certificatePath) - if err != nil { - return nil, nil, errors.Wrapf(err, "couldn't load the certificate file %s", certificatePath) - } - - cert := certs[0] - intermediates := certs[1:] - - return cert, intermediates, nil -} - -// VerifyCertChain verifies that a certificate has a valid chain of -// intermediate CAs back to the root CA -func VerifyCertChain(cert *x509.Certificate, intermediates []*x509.Certificate, root *x509.Certificate) error { - rootPool := x509.NewCertPool() - rootPool.AddCert(root) - - intermediatePool := x509.NewCertPool() - for _, c := range intermediates { - intermediatePool.AddCert(c) - } - - verifyOptions := x509.VerifyOptions{ - Roots: rootPool, - Intermediates: intermediatePool, - KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, - } - - if _, err := cert.Verify(verifyOptions); err != nil { - return err - } - - return nil -} - -// TryLoadCertAndKeyFromDisk tries to load a cert and a key from the disk and validates that they are valid -func TryLoadCertAndKeyFromDisk(pkiPath, name string) (*x509.Certificate, crypto.Signer, error) { - cert, err := TryLoadCertFromDisk(pkiPath, name) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to load certificate") - } - - key, err := TryLoadKeyFromDisk(pkiPath, name) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to load key") - } - - return cert, key, nil -} - -// TryLoadCertFromDisk tries to load the cert from the disk -func TryLoadCertFromDisk(pkiPath, name string) (*x509.Certificate, error) { - certificatePath := pathForCert(pkiPath, name) - - certs, err := certutil.CertsFromFile(certificatePath) - if err != nil { - return nil, errors.Wrapf(err, "couldn't load the certificate file %s", certificatePath) - } - - // We are only putting one certificate in the certificate pem file, so it's safe to just pick the first one - // TODO: Support multiple certs here in order to be able to rotate certs - cert := certs[0] - - return cert, nil -} - -// CertConfig is a wrapper around certutil.Config extending it with PublicKeyAlgorithm. -type CertConfig struct { - certutil.Config - NotAfter *time.Time - PublicKeyAlgorithm x509.PublicKeyAlgorithm -} - -var ( - // certPeriodValidation is used to store if period validation was done for a certificate - certPeriodValidationMutex sync.Mutex - certPeriodValidation = map[string]struct{}{} -) - -// NewPrivateKey returns a new private key. -var NewPrivateKey = GeneratePrivateKey - -func GeneratePrivateKey(keyType x509.PublicKeyAlgorithm) (crypto.Signer, error) { - if keyType == x509.ECDSA { - return ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader) - } - - return rsa.GenerateKey(cryptorand.Reader, rsaKeySize) -} - -// NewCertificateAuthority creates new certificate and private key for the certificate authority -func NewCertificateAuthority(config *CertConfig) (*x509.Certificate, crypto.Signer, error) { - key, err := NewPrivateKey(config.PublicKeyAlgorithm) - if err != nil { - return nil, nil, errors.Wrap(err, "unable to create private key while generating CA certificate") - } - - cert, err := certutil.NewSelfSignedCACert(config.Config, key) - if err != nil { - return nil, nil, errors.Wrap(err, "unable to create self-signed CA certificate") - } - - return cert, key, nil -} - -// writeCertificateAuthorityFilesIfNotExist write a new certificate Authority to the given path. -// If there already is a certificate file at the given path; kubeadm tries to load it and check if the values in the -// existing and the expected certificate equals. If they do; kubeadm will just skip writing the file as it's up-to-date, -// otherwise this function returns an error. -func writeCertificateAuthorityFilesIfNotExist(pkiDir string, baseName string, caCert *x509.Certificate, caKey crypto.Signer) error { - - // If cert or key exists, we should try to load them - if CertOrKeyExist(pkiDir, baseName) { - - // Try to load .crt and .key from the PKI directory - caCert, _, err := TryLoadCertAndKeyFromDisk(pkiDir, baseName) - if err != nil { - return errors.Wrapf(err, "failure loading %s certificate", baseName) - } - // Validate period - CheckCertificatePeriodValidity(baseName, caCert) - - // Check if the existing cert is a CA - if !caCert.IsCA { - return errors.Errorf("certificate %s is not a CA", baseName) - } - - // kubeadm doesn't validate the existing certificate Authority more than this; - // Basically, if we find a certificate file with the same path; and it is a CA - // kubeadm thinks those files are equal and doesn't bother writing a new file - fmt.Printf("[certs] Using the existing %q certificate and key\n", baseName) - } else { - // Write .crt and .key files to disk - fmt.Printf("[certs] Generating %q certificate and key\n", baseName) - - if err := WriteCertAndKey(pkiDir, baseName, caCert, caKey); err != nil { - return errors.Wrapf(err, "failure while saving %s certificate and key", baseName) - } - } - return nil -} - -// validateCertificateWithConfig makes sure that a given certificate is valid at -// least for the SANs defined in the configuration. -func validateCertificateWithConfig(cert *x509.Certificate, baseName string, cfg *CertConfig) error { - for _, dnsName := range cfg.AltNames.DNSNames { - if err := cert.VerifyHostname(dnsName); err != nil { - return errors.Wrapf(err, "certificate %s is invalid", baseName) - } - } - for _, ipAddress := range cfg.AltNames.IPs { - if err := cert.VerifyHostname(ipAddress.String()); err != nil { - return errors.Wrapf(err, "certificate %s is invalid", baseName) - } - } - return nil -} - -// WriteCertAndKey stores certificate and key at the specified location -func WriteCertAndKey(pkiPath string, name string, cert *x509.Certificate, key crypto.Signer) error { - if err := WriteKey(pkiPath, name, key); err != nil { - return errors.Wrap(err, "couldn't write key") - } - - return WriteCert(pkiPath, name, cert) -} - -// HasServerAuth returns true if the given certificate is a ServerAuth -func HasServerAuth(cert *x509.Certificate) bool { - for i := range cert.ExtKeyUsage { - if cert.ExtKeyUsage[i] == x509.ExtKeyUsageServerAuth { - return true - } - } - return false -} - -// ValidateCertPeriod checks if the certificate is valid relative to the current time -// (+/- offset) -func ValidateCertPeriod(cert *x509.Certificate, offset time.Duration) error { - period := fmt.Sprintf("NotBefore: %v, NotAfter: %v", cert.NotBefore, cert.NotAfter) - now := time.Now().Add(offset) - if now.Before(cert.NotBefore) { - return errors.Errorf("the certificate is not valid yet: %s", period) - } - if now.After(cert.NotAfter) { - return errors.Errorf("the certificate has expired: %s", period) - } - return nil -} - -// WriteKey stores the given key at the given location -func WriteKey(pkiPath, name string, key crypto.Signer) error { - if key == nil { - return errors.New("private key cannot be nil when writing to file") - } - - privateKeyPath := pathForKey(pkiPath, name) - encoded, err := keyutil.MarshalPrivateKeyToPEM(key) - if err != nil { - return errors.Wrapf(err, "unable to marshal private key to PEM") - } - if err := keyutil.WriteKey(privateKeyPath, encoded); err != nil { - return errors.Wrapf(err, "unable to write private key to file %s", privateKeyPath) - } - - return nil -} - -// WriteCert stores the given certificate at the given location -func WriteCert(pkiPath, name string, cert *x509.Certificate) error { - if cert == nil { - return errors.New("certificate cannot be nil when writing to file") - } - - certificatePath := pathForCert(pkiPath, name) - if err := certutil.WriteCert(certificatePath, EncodeCertPEM(cert)); err != nil { - return errors.Wrapf(err, "unable to write certificate to file %s", certificatePath) - } - - return nil -} - -// EncodeCertPEM returns PEM-endcoded certificate data -func EncodeCertPEM(cert *x509.Certificate) []byte { - block := pem.Block{ - Type: CertificateBlockType, - Bytes: cert.Raw, - } - return pem.EncodeToMemory(&block) -} diff --git a/cmd/kk/pkg/utils/client.go b/cmd/kk/pkg/utils/client.go deleted file mode 100644 index 98f22cfa..00000000 --- a/cmd/kk/pkg/utils/client.go +++ /dev/null @@ -1,58 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package utils - -import ( - "os" - "path/filepath" - - "github.com/pkg/errors" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" -) - -func NewClient(config string) (*kubernetes.Clientset, error) { - var kubeconfig string - if config != "" { - config, err := filepath.Abs(config) - if err != nil { - return nil, errors.Wrap(err, "Failed to look up current directory") - } - kubeconfig = config - } else { - kubeconfig = filepath.Join(homeDir(), ".kube", "config") - } - // use the current context in kubeconfig - configCluster, err := clientcmd.BuildConfigFromFlags("", kubeconfig) - if err != nil { - return nil, err - } - // create the clientset - clientset, err := kubernetes.NewForConfig(configCluster) - if err != nil { - return nil, err - } - - return clientset, nil -} - -func homeDir() string { - if h := os.Getenv("HOME"); h != "" { - return h - } - return os.Getenv("USERPROFILE") -} diff --git a/cmd/kk/pkg/utils/containerruntime/runtime.go b/cmd/kk/pkg/utils/containerruntime/runtime.go deleted file mode 100644 index 46a0dafd..00000000 --- a/cmd/kk/pkg/utils/containerruntime/runtime.go +++ /dev/null @@ -1,82 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package containerruntime - -import ( - "os" - "strings" - - "github.com/pkg/errors" -) - -const ( - dockerSocket = "/var/run/docker.sock" // The Docker socket is not CRI compatible - - DefaultDockerCRISocket = "/var/run/dockershim.sock" - containerdSocket = "/run/containerd/containerd.sock" -) - -// isExistingSocket checks if path exists and is domain socket -func isExistingSocket(path string) bool { - fileInfo, err := os.Stat(path) - if err != nil { - return false - } - - return fileInfo.Mode()&os.ModeSocket != 0 -} - -// detectCRISocketImpl is separated out only for test purposes, DON'T call it directly, use DetectCRISocket instead -func detectCRISocketImpl(isSocket func(string) bool) (string, error) { - foundCRISockets := []string{} - knownCRISockets := []string{ - // Docker and containerd sockets are special cased below, hence not to be included here - "/var/run/crio/crio.sock", - } - - if isSocket(containerdSocket) { - // Docker 18.09 gets bundled together with containerd, thus having both dockerSocket and containerdSocket present. - // Docker will be deprecated, so prefer to use containerd. - foundCRISockets = append(foundCRISockets, containerdSocket) - } else if isSocket(dockerSocket) { - // the path in dockerSocket is not CRI compatible, hence we should replace it with a CRI compatible socket - foundCRISockets = append(foundCRISockets, DefaultDockerCRISocket) - } - - for _, socket := range knownCRISockets { - if isSocket(socket) { - foundCRISockets = append(foundCRISockets, socket) - } - } - - switch len(foundCRISockets) { - case 0: - // Fall back to Docker if no CRI is detected, we can error out later on if we need it - return containerdSocket, nil - case 1: - // Precisely one CRI found, use that - return foundCRISockets[0], nil - default: - // Multiple CRIs installed? - return "", errors.Errorf("Found multiple CRI sockets, please use --cri-socket to select one: %s", strings.Join(foundCRISockets, ", ")) - } -} - -// DetectCRISocket uses a list of known CRI sockets to detect one. If more than one or none is discovered, an error is returned. -func DetectCRISocket() (string, error) { - return detectCRISocketImpl(isExistingSocket) -} diff --git a/cmd/kk/pkg/utils/util.go b/cmd/kk/pkg/utils/util.go deleted file mode 100644 index e77b4851..00000000 --- a/cmd/kk/pkg/utils/util.go +++ /dev/null @@ -1,57 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package utils - -import ( - "fmt" - "regexp" - "strings" - "text/template" - - "github.com/pkg/errors" - "gopkg.in/yaml.v3" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" -) - -var FuncMap = template.FuncMap{"toYaml": ToYAML, "indent": Indent} - -func ResetTmpDir(runtime connector.Runtime) error { - _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf( - "if [ -d %s ]; then rm -rf %s ;fi && mkdir -m 777 -p %s", - common.TmpDir, common.TmpDir, common.TmpDir), false) - if err != nil { - return errors.Wrap(errors.WithStack(err), "reset tmp dir failed") - } - return nil -} - -func ToYAML(v interface{}) string { - data, err := yaml.Marshal(v) - if err != nil { - // Swallow errors inside of a template. - return "" - } - return strings.TrimSuffix(string(data), "\n") -} - -func Indent(n int, text string) string { - startOfLine := regexp.MustCompile(`(?m)^`) - indentation := strings.Repeat(" ", n) - return strings.TrimRight(startOfLine.ReplaceAllLiteralString(text, indentation), " ") -} diff --git a/cmd/kk/pkg/version/kubernetes/version_enum.go b/cmd/kk/pkg/version/kubernetes/version_enum.go deleted file mode 100644 index e0fffdbb..00000000 --- a/cmd/kk/pkg/version/kubernetes/version_enum.go +++ /dev/null @@ -1,105 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "fmt" - "sort" - - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" -) - -type Version int - -const ( - V119 Version = iota - V120 - V121 - V122 - V123 - V124 - V125 - V126 - V127 - V128 -) - -var VersionList = []Version{ - V119, - V120, - V121, - V122, - V123, - V124, - V125, - V126, - V127, - V128, -} - -func (v Version) String() string { - switch v { - case V119: - return "v1.19" - case V120: - return "v1.20" - case V121: - return "v1.21" - case V122: - return "v1.22" - case V123: - return "v1.23" - case V124: - return "v1.24" - case V125: - return "v1.25" - case V126: - return "v1.26" - case V127: - return "v1.27" - case V128: - return "v1.28" - default: - return "invalid option" - } -} - -func VersionSupport(version string) bool { - K8sTargetVersion := versionutil.MustParseSemantic(version) - for i := range VersionList { - if VersionList[i].String() == fmt.Sprintf("v%v.%v", K8sTargetVersion.Major(), K8sTargetVersion.Minor()) { - return true - } - } - return false -} - -// SupportedK8sVersionList returns the supported list of Kubernetes -func SupportedK8sVersionList() []string { - - versionsList := make([]string, 0, len(files.FileSha256["kubeadm"]["amd64"])) - for k := range files.FileSha256["kubeadm"]["amd64"] { - versionsList = append(versionsList, k) - } - sort.Slice(versionsList, func(i, j int) bool { - return versionutil.MustParseSemantic(versionsList[i]).LessThan(versionutil.MustParseSemantic(versionsList[j])) - }) - - return versionsList -} diff --git a/cmd/kk/pkg/version/kubesphere/ks_installer.go b/cmd/kk/pkg/version/kubesphere/ks_installer.go deleted file mode 100644 index 22719db1..00000000 --- a/cmd/kk/pkg/version/kubesphere/ks_installer.go +++ /dev/null @@ -1,260 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubesphere - -import ( - "fmt" - "os" - "text/template" - - versionutil "k8s.io/apimachinery/pkg/util/version" - - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" - "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubesphere/templates" -) - -type KsInstaller struct { - Version string - CRDTemplate *template.Template - ClusterConfigurationTemplate *template.Template - K8sSupportVersions []string - UpgradeSupportVersions []string -} - -func (k *KsInstaller) CCToString() string { - str, err := util.Render(k.ClusterConfigurationTemplate, util.Data{ - "Tag": k.Version, - }) - if err != nil { - os.Exit(0) - } - return str -} - -func (k *KsInstaller) K8sSupport(version string) bool { - k8sVersion := versionutil.MustParseSemantic(version) - for i := range k.K8sSupportVersions { - if k.K8sSupportVersions[i] == fmt.Sprintf("v%v.%v", k8sVersion.Major(), k8sVersion.Minor()) { - return true - } - } - return false -} - -func (k *KsInstaller) UpgradeSupport(version string) bool { - for i := range k.UpgradeSupportVersions { - if k.UpgradeSupportVersions[i] == version { - return true - } - } - return false -} - -var KsV211 = &KsInstaller{ - Version: V211.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V211, - K8sSupportVersions: []string{ - "v1.15", - "v1.16", - "v1.17", - "v1.18", - }, - UpgradeSupportVersions: []string{ - V211.String(), - }, -} - -var KsV300 = &KsInstaller{ - Version: V300.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V300, - K8sSupportVersions: []string{ - "v1.15", - "v1.16", - "v1.17", - "v1.18", - }, - UpgradeSupportVersions: []string{ - V211.String(), - }, -} - -var KsV310 = &KsInstaller{ - Version: V310.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V310, - K8sSupportVersions: []string{ - "v1.15", - "v1.16", - "v1.17", - "v1.18", - "v1.19", - "v1.20", - }, - UpgradeSupportVersions: []string{ - V300.String(), - }, -} - -var KsV311 = &KsInstaller{ - Version: V311.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V311, - K8sSupportVersions: []string{ - "v1.15", - "v1.16", - "v1.17", - "v1.18", - "v1.19", - "v1.20", - }, - UpgradeSupportVersions: []string{ - V300.String(), - V310.String(), - }, -} - -var KsV320 = &KsInstaller{ - Version: V320.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V320, - K8sSupportVersions: []string{ - "v1.19", - "v1.20", - "v1.21", - "v1.22", - }, - UpgradeSupportVersions: []string{ - V310.String(), - V311.String(), - }, -} - -var KsV321 = &KsInstaller{ - Version: V321.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V321, - K8sSupportVersions: []string{ - "v1.19", - "v1.20", - "v1.21", - "v1.22", - }, - UpgradeSupportVersions: []string{ - V310.String(), - V311.String(), - V320.String(), - }, -} - -var KsV330 = &KsInstaller{ - Version: V330.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V330, - K8sSupportVersions: []string{ - "v1.19", - "v1.20", - "v1.21", - "v1.22", - "v1.23", - }, - UpgradeSupportVersions: []string{ - V320.String(), - V321.String(), - }, -} - -var KsV331 = &KsInstaller{ - Version: V331.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V331, - K8sSupportVersions: []string{ - "v1.19", - "v1.20", - "v1.21", - "v1.22", - "v1.23", - "v1.24", - }, - UpgradeSupportVersions: []string{ - V330.String(), - V320.String(), - V321.String(), - }, -} - -var KsV332 = &KsInstaller{ - Version: V332.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V332, - K8sSupportVersions: []string{ - "v1.19", - "v1.20", - "v1.21", - "v1.22", - "v1.23", - "v1.24", - }, - UpgradeSupportVersions: []string{ - V331.String(), - V330.String(), - V320.String(), - V321.String(), - }, -} - -var KsV340 = &KsInstaller{ - Version: V340.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V340, - K8sSupportVersions: []string{ - "v1.19", - "v1.20", - "v1.21", - "v1.22", - "v1.23", - "v1.24", - "v1.25", - "v1.26", - }, - UpgradeSupportVersions: []string{ - V332.String(), - V331.String(), - V330.String(), - }, -} - -var KsV341 = &KsInstaller{ - Version: V341.String(), - CRDTemplate: templates.KsInstaller, - ClusterConfigurationTemplate: templates.V341, - K8sSupportVersions: []string{ - "v1.21", - "v1.22", - "v1.23", - "v1.24", - "v1.25", - "v1.26", - }, - UpgradeSupportVersions: []string{ - V340.String(), - V332.String(), - V331.String(), - V330.String(), - }, -} diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v211.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v211.go deleted file mode 100644 index f724df06..00000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v211.go +++ /dev/null @@ -1,92 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var V211 = template.Must(template.New("v2.1.1").Parse( - dedent.Dedent(`--- -apiVersion: v1 -data: - ks-config.yaml: | - --- - local_registry: "" - persistence: - storageClass: "" - etcd: - monitoring: true - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - mysqlVolumeSize: 20Gi - minioVolumeSize: 20Gi - etcdVolumeSize: 20Gi - openldapVolumeSize: 2Gi - redisVolumSize: 2Gi - metrics_server: - enabled: false - console: - enableMultiLogin: False # enable/disable multi login - port: 30880 - monitoring: - prometheusReplicas: 1 - prometheusMemoryRequest: 400Mi - prometheusVolumeSize: 20Gi - grafana: - enabled: false - logging: - enabled: false - elasticsearchMasterReplicas: 1 - elasticsearchDataReplicas: 1 - logsidecarReplicas: 2 - elasticsearchMasterVolumeSize: 4Gi - elasticsearchDataVolumeSize: 20Gi - logMaxAge: 7 - elkPrefix: logstash - containersLogMountedPath: "" - kibana: - enabled: false - openpitrix: - enabled: false - devops: - enabled: false - jenkinsMemoryLim: 2Gi - jenkinsMemoryReq: 1500Mi - jenkinsVolumeSize: 8Gi - jenkinsJavaOpts_Xms: 512m - jenkinsJavaOpts_Xmx: 512m - jenkinsJavaOpts_MaxRAM: 2g - sonarqube: - enabled: false - postgresqlVolumeSize: 8Gi - servicemesh: - enabled: false - notification: - enabled: false - alerting: - enabled: false -kind: ConfigMap -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v300.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v300.go deleted file mode 100644 index 91bd178a..00000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v300.go +++ /dev/null @@ -1,94 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var V300 = template.Must(template.New("v3.0.0").Parse( - dedent.Dedent(`--- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - zone: "" - local_registry: "" - persistence: - storageClass: "" - authentication: - jwtSecret: "" - etcd: - monitoring: true - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - es: - elasticsearchDataVolumeSize: 20Gi - elasticsearchMasterVolumeSize: 4Gi - elkPrefix: logstash - logMaxAge: 7 - mysqlVolumeSize: 20Gi - minioVolumeSize: 20Gi - etcdVolumeSize: 20Gi - openldapVolumeSize: 2Gi - redisVolumSize: 2Gi - console: - enableMultiLogin: false # enable/disable multi login - port: 30880 - alerting: - enabled: false - auditing: - enabled: false - devops: - enabled: false - jenkinsMemoryLim: 2Gi - jenkinsMemoryReq: 1500Mi - jenkinsVolumeSize: 8Gi - jenkinsJavaOpts_Xms: 512m - jenkinsJavaOpts_Xmx: 512m - jenkinsJavaOpts_MaxRAM: 2g - events: - enabled: false - ruler: - enabled: true - replicas: 2 - logging: - enabled: false - logsidecarReplicas: 2 - metrics_server: - enabled: true - monitoring: - prometheusMemoryRequest: 400Mi - prometheusVolumeSize: 20Gi - multicluster: - clusterRole: none # host | member | none - networkpolicy: - enabled: false - notification: - enabled: false - openpitrix: - enabled: false - servicemesh: - enabled: false -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v310.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v310.go deleted file mode 100644 index e7d14cc9..00000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v310.go +++ /dev/null @@ -1,140 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var V310 = template.Must(template.New("v3.1.0").Parse( - dedent.Dedent(`--- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - zone: "" - local_registry: "" - etcd: - monitoring: false - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - redis: - enabled: false - redisVolumSize: 2Gi - openldap: - enabled: false - openldapVolumeSize: 2Gi - minioVolumeSize: 20Gi - monitoring: - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - es: - elasticsearchMasterVolumeSize: 4Gi - elasticsearchDataVolumeSize: 20Gi - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchUrl: "" - externalElasticsearchPort: "" - console: - enableMultiLogin: true - port: 30880 - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - devops: - enabled: false - jenkinsMemoryLim: 2Gi - jenkinsMemoryReq: 1500Mi - jenkinsVolumeSize: 8Gi - jenkinsJavaOpts_Xms: 512m - jenkinsJavaOpts_Xmx: 512m - jenkinsJavaOpts_MaxRAM: 2g - events: - enabled: false - ruler: - enabled: true - replicas: 2 - logging: - enabled: false - logsidecar: - enabled: true - replicas: 2 - metrics_server: - enabled: false - monitoring: - storageClass: "" - prometheusMemoryRequest: 400Mi - prometheusVolumeSize: 20Gi - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - kubeedge: - enabled: false - cloudCore: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] - cloudhubPort: "10000" - cloudhubQuicPort: "10001" - cloudhubHttpsPort: "10002" - cloudstreamPort: "10003" - tunnelPort: "10004" - cloudHub: - advertiseAddress: - - "" - nodeLimit: "100" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - edgeWatcher: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] - edgeWatcherAgent: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v311.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v311.go deleted file mode 100644 index dc5ebe31..00000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v311.go +++ /dev/null @@ -1,140 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var V311 = template.Must(template.New("v3.1.1").Parse( - dedent.Dedent(`--- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - zone: "" - local_registry: "" - etcd: - monitoring: false - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - redis: - enabled: false - redisVolumSize: 2Gi - openldap: - enabled: false - openldapVolumeSize: 2Gi - minioVolumeSize: 20Gi - monitoring: - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - es: - elasticsearchMasterVolumeSize: 4Gi - elasticsearchDataVolumeSize: 20Gi - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchUrl: "" - externalElasticsearchPort: "" - console: - enableMultiLogin: true - port: 30880 - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - devops: - enabled: false - jenkinsMemoryLim: 2Gi - jenkinsMemoryReq: 1500Mi - jenkinsVolumeSize: 8Gi - jenkinsJavaOpts_Xms: 512m - jenkinsJavaOpts_Xmx: 512m - jenkinsJavaOpts_MaxRAM: 2g - events: - enabled: false - ruler: - enabled: true - replicas: 2 - logging: - enabled: false - logsidecar: - enabled: true - replicas: 2 - metrics_server: - enabled: false - monitoring: - storageClass: "" - prometheusMemoryRequest: 400Mi - prometheusVolumeSize: 20Gi - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - kubeedge: - enabled: false - cloudCore: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] - cloudhubPort: "10000" - cloudhubQuicPort: "10001" - cloudhubHttpsPort: "10002" - cloudstreamPort: "10003" - tunnelPort: "10004" - cloudHub: - advertiseAddress: - - "" - nodeLimit: "100" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - edgeWatcher: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] - edgeWatcherAgent: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v320.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v320.go deleted file mode 100644 index e3720966..00000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v320.go +++ /dev/null @@ -1,199 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var V320 = template.Must(template.New("v3.2.0").Parse( - dedent.Dedent(` ---- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - zone: "" - local_registry: "" - # dev_tag: "" - etcd: - monitoring: false - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - core: - console: - enableMultiLogin: true - port: 30880 - type: NodePort - # apiserver: - # resources: {} - # controllerManager: - # resources: {} - redis: - enabled: false - volumeSize: 2Gi - openldap: - enabled: false - volumeSize: 2Gi - minio: - volumeSize: 20Gi - monitoring: - # type: external - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - GPUMonitoring: - enabled: false - gpu: - kinds: - - resourceName: "nvidia.com/gpu" - resourceType: "GPU" - default: true - es: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchUrl: "" - externalElasticsearchPort: "" - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - # operator: - # resources: {} - # webhook: - # resources: {} - devops: - enabled: false - jenkinsMemoryLim: 2Gi - jenkinsMemoryReq: 1500Mi - jenkinsVolumeSize: 8Gi - jenkinsJavaOpts_Xms: 512m - jenkinsJavaOpts_Xmx: 512m - jenkinsJavaOpts_MaxRAM: 2g - events: - enabled: false - # operator: - # resources: {} - # exporter: - # resources: {} - # ruler: - # enabled: true - # replicas: 2 - # resources: {} - logging: - enabled: false - containerruntime: docker - logsidecar: - enabled: true - replicas: 2 - # resources: {} - metrics_server: - enabled: false - monitoring: - storageClass: "" - # kube_rbac_proxy: - # resources: {} - # kube_state_metrics: - # resources: {} - # prometheus: - # replicas: 1 - # volumeSize: 20Gi - # resources: {} - # operator: - # resources: {} - # adapter: - # resources: {} - # node_exporter: - # resources: {} - # alertmanager: - # replicas: 1 - # resources: {} - # notification_manager: - # resources: {} - # operator: - # resources: {} - # proxy: - # resources: {} - gpu: - nvidia_dcgm_exporter: - enabled: false - # resources: {} - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - kubeedge: - enabled: false - cloudCore: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] - cloudhubPort: "10000" - cloudhubQuicPort: "10001" - cloudhubHttpsPort: "10002" - cloudstreamPort: "10003" - tunnelPort: "10004" - cloudHub: - advertiseAddress: - - "" - nodeLimit: "100" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - edgeWatcher: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] - edgeWatcherAgent: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v321.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v321.go deleted file mode 100644 index 5fc4f35d..00000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v321.go +++ /dev/null @@ -1,200 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var V321 = template.Must(template.New("v3.2.1").Parse( - dedent.Dedent(` ---- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - zone: "" - local_registry: "" - namespace_override: "" - # dev_tag: "" - etcd: - monitoring: true - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - core: - console: - enableMultiLogin: true - port: 30880 - type: NodePort - # apiserver: - # resources: {} - # controllerManager: - # resources: {} - redis: - enabled: false - volumeSize: 2Gi - openldap: - enabled: false - volumeSize: 2Gi - minio: - volumeSize: 20Gi - monitoring: - # type: external - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - GPUMonitoring: - enabled: false - gpu: - kinds: - - resourceName: "nvidia.com/gpu" - resourceType: "GPU" - default: true - es: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchHost: "" - externalElasticsearchPort: "" - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - # operator: - # resources: {} - # webhook: - # resources: {} - devops: - enabled: false - jenkinsMemoryLim: 2Gi - jenkinsMemoryReq: 1500Mi - jenkinsVolumeSize: 8Gi - jenkinsJavaOpts_Xms: 512m - jenkinsJavaOpts_Xmx: 512m - jenkinsJavaOpts_MaxRAM: 2g - events: - enabled: false - # operator: - # resources: {} - # exporter: - # resources: {} - # ruler: - # enabled: true - # replicas: 2 - # resources: {} - logging: - enabled: false - containerruntime: docker - logsidecar: - enabled: true - replicas: 2 - # resources: {} - metrics_server: - enabled: false - monitoring: - storageClass: "" - # kube_rbac_proxy: - # resources: {} - # kube_state_metrics: - # resources: {} - # prometheus: - # replicas: 1 - # volumeSize: 20Gi - # resources: {} - # operator: - # resources: {} - # adapter: - # resources: {} - # node_exporter: - # resources: {} - # alertmanager: - # replicas: 1 - # resources: {} - # notification_manager: - # resources: {} - # operator: - # resources: {} - # proxy: - # resources: {} - gpu: - nvidia_dcgm_exporter: - enabled: false - # resources: {} - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - kubeedge: - enabled: false - cloudCore: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] - cloudhubPort: "10000" - cloudhubQuicPort: "10001" - cloudhubHttpsPort: "10002" - cloudstreamPort: "10003" - tunnelPort: "10004" - cloudHub: - advertiseAddress: - - "" - nodeLimit: "100" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - edgeWatcher: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] - edgeWatcherAgent: - nodeSelector: {"node-role.kubernetes.io/worker": ""} - tolerations: [] -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v330.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v330.go deleted file mode 100644 index a5561417..00000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v330.go +++ /dev/null @@ -1,204 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "github.com/lithammer/dedent" - "text/template" -) - -var V330 = template.Must(template.New("v3.3.0").Parse( - dedent.Dedent(` ---- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - zone: "" - local_registry: "" - namespace_override: "" - # dev_tag: "" - etcd: - monitoring: false - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - core: - console: - enableMultiLogin: true - port: 30880 - type: NodePort - # apiserver: - # resources: {} - # controllerManager: - # resources: {} - redis: - enabled: false - volumeSize: 2Gi - openldap: - enabled: false - volumeSize: 2Gi - minio: - volumeSize: 20Gi - monitoring: - # type: external - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - GPUMonitoring: - enabled: false - gpu: - kinds: - - resourceName: "nvidia.com/gpu" - resourceType: "GPU" - default: true - es: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchHost: "" - externalElasticsearchPort: "" - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - # operator: - # resources: {} - # webhook: - # resources: {} - devops: - enabled: false - # resources: {} - jenkinsMemoryLim: 2Gi - jenkinsMemoryReq: 1500Mi - jenkinsVolumeSize: 8Gi - jenkinsJavaOpts_Xms: 1200m - jenkinsJavaOpts_Xmx: 1600m - jenkinsJavaOpts_MaxRAM: 2g - events: - enabled: false - # operator: - # resources: {} - # exporter: - # resources: {} - # ruler: - # enabled: true - # replicas: 2 - # resources: {} - logging: - enabled: false - logsidecar: - enabled: true - replicas: 2 - # resources: {} - metrics_server: - enabled: false - monitoring: - storageClass: "" - node_exporter: - port: 9100 - # resources: {} - # kube_rbac_proxy: - # resources: {} - # kube_state_metrics: - # resources: {} - # prometheus: - # replicas: 1 - # volumeSize: 20Gi - # resources: {} - # operator: - # resources: {} - # alertmanager: - # replicas: 1 - # resources: {} - # notification_manager: - # resources: {} - # operator: - # resources: {} - # proxy: - # resources: {} - gpu: - nvidia_dcgm_exporter: - enabled: false - # resources: {} - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - istio: - components: - ingressGateways: - - name: istio-ingressgateway - enabled: false - cni: - enabled: false - edgeruntime: - enabled: false - kubeedge: - enabled: false - cloudCore: - cloudHub: - advertiseAddress: - - "" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - # resources: {} - # hostNetWork: false - iptables-manager: - enabled: true - mode: "external" - # resources: {} - # edgeService: - # resources: {} - terminal: - timeout: 600 -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v331.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v331.go deleted file mode 100644 index 1b1b0e5b..00000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v331.go +++ /dev/null @@ -1,202 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var V331 = template.Must(template.New("v3.3.1").Parse( - dedent.Dedent(` ---- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - zone: "" - local_registry: "" - namespace_override: "" - # dev_tag: "" - etcd: - monitoring: false - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - core: - console: - enableMultiLogin: true - port: 30880 - type: NodePort - # apiserver: - # resources: {} - # controllerManager: - # resources: {} - redis: - enabled: false - volumeSize: 2Gi - openldap: - enabled: false - volumeSize: 2Gi - minio: - volumeSize: 20Gi - monitoring: - # type: external - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - GPUMonitoring: - enabled: false - gpu: - kinds: - - resourceName: "nvidia.com/gpu" - resourceType: "GPU" - default: true - es: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchHost: "" - externalElasticsearchPort: "" - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - # operator: - # resources: {} - # webhook: - # resources: {} - devops: - enabled: false - # resources: {} - jenkinsMemoryLim: 8Gi - jenkinsMemoryReq: 4Gi - jenkinsVolumeSize: 8Gi - events: - enabled: false - # operator: - # resources: {} - # exporter: - # resources: {} - # ruler: - # enabled: true - # replicas: 2 - # resources: {} - logging: - enabled: false - logsidecar: - enabled: true - replicas: 2 - # resources: {} - metrics_server: - enabled: false - monitoring: - storageClass: "" - node_exporter: - port: 9100 - # resources: {} - # kube_rbac_proxy: - # resources: {} - # kube_state_metrics: - # resources: {} - # prometheus: - # replicas: 1 - # volumeSize: 20Gi - # resources: {} - # operator: - # resources: {} - # alertmanager: - # replicas: 1 - # resources: {} - # notification_manager: - # resources: {} - # operator: - # resources: {} - # proxy: - # resources: {} - gpu: - nvidia_dcgm_exporter: - enabled: false - # resources: {} - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - istio: - components: - ingressGateways: - - name: istio-ingressgateway - enabled: false - cni: - enabled: false - edgeruntime: - enabled: false - kubeedge: - enabled: false - cloudCore: - cloudHub: - advertiseAddress: - - "" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - # resources: {} - # hostNetWork: false - iptables-manager: - enabled: true - mode: "external" - # resources: {} - # edgeService: - # resources: {} - terminal: - timeout: 600 -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v332.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v332.go deleted file mode 100644 index 7463d1c5..00000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v332.go +++ /dev/null @@ -1,202 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var V332 = template.Must(template.New("v3.3.2").Parse( - dedent.Dedent(` ---- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - zone: "" - local_registry: "" - namespace_override: "" - # dev_tag: "" - etcd: - monitoring: false - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - core: - console: - enableMultiLogin: true - port: 30880 - type: NodePort - # apiserver: - # resources: {} - # controllerManager: - # resources: {} - redis: - enabled: false - volumeSize: 2Gi - openldap: - enabled: false - volumeSize: 2Gi - minio: - volumeSize: 20Gi - monitoring: - # type: external - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - GPUMonitoring: - enabled: false - gpu: - kinds: - - resourceName: "nvidia.com/gpu" - resourceType: "GPU" - default: true - es: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchHost: "" - externalElasticsearchPort: "" - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - # operator: - # resources: {} - # webhook: - # resources: {} - devops: - enabled: false - # resources: {} - jenkinsMemoryLim: 8Gi - jenkinsMemoryReq: 4Gi - jenkinsVolumeSize: 8Gi - events: - enabled: false - # operator: - # resources: {} - # exporter: - # resources: {} - # ruler: - # enabled: true - # replicas: 2 - # resources: {} - logging: - enabled: false - logsidecar: - enabled: true - replicas: 2 - # resources: {} - metrics_server: - enabled: false - monitoring: - storageClass: "" - node_exporter: - port: 9100 - # resources: {} - # kube_rbac_proxy: - # resources: {} - # kube_state_metrics: - # resources: {} - # prometheus: - # replicas: 1 - # volumeSize: 20Gi - # resources: {} - # operator: - # resources: {} - # alertmanager: - # replicas: 1 - # resources: {} - # notification_manager: - # resources: {} - # operator: - # resources: {} - # proxy: - # resources: {} - gpu: - nvidia_dcgm_exporter: - enabled: false - # resources: {} - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - istio: - components: - ingressGateways: - - name: istio-ingressgateway - enabled: false - cni: - enabled: false - edgeruntime: - enabled: false - kubeedge: - enabled: false - cloudCore: - cloudHub: - advertiseAddress: - - "" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - # resources: {} - # hostNetWork: false - iptables-manager: - enabled: true - mode: "external" - # resources: {} - # edgeService: - # resources: {} - terminal: - timeout: 600 -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v340.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v340.go deleted file mode 100644 index b76fe1fc..00000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v340.go +++ /dev/null @@ -1,230 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var V340 = template.Must(template.New("v3.4.0").Parse( - dedent.Dedent(` ---- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - zone: "" - local_registry: "" - namespace_override: "" - # dev_tag: "" - etcd: - monitoring: false - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - core: - console: - enableMultiLogin: true - port: 30880 - type: NodePort - # apiserver: - # resources: {} - # controllerManager: - # resources: {} - redis: - enabled: false - enableHA: false - volumeSize: 2Gi - openldap: - enabled: false - volumeSize: 2Gi - minio: - volumeSize: 20Gi - monitoring: - # type: external - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - GPUMonitoring: - enabled: false - gpu: - kinds: - - resourceName: "nvidia.com/gpu" - resourceType: "GPU" - default: true - es: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchHost: "" - externalElasticsearchPort: "" - opensearch: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - enabled: true - logMaxAge: 7 - opensearchPrefix: whizard - basicAuth: - enabled: true - username: "admin" - password: "admin" - externalOpensearchHost: "" - externalOpensearchPort: "" - dashboard: - enabled: false - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - # operator: - # resources: {} - # webhook: - # resources: {} - devops: - enabled: false - jenkinsCpuReq: 0.5 - jenkinsCpuLim: 1 - jenkinsMemoryReq: 4Gi - jenkinsMemoryLim: 4Gi - jenkinsVolumeSize: 16Gi - events: - enabled: false - # operator: - # resources: {} - # exporter: - # resources: {} - # ruler: - # enabled: true - # replicas: 2 - # resources: {} - logging: - enabled: false - logsidecar: - enabled: true - replicas: 2 - # resources: {} - metrics_server: - enabled: false - monitoring: - storageClass: "" - node_exporter: - port: 9100 - # resources: {} - # kube_rbac_proxy: - # resources: {} - # kube_state_metrics: - # resources: {} - # prometheus: - # replicas: 1 - # volumeSize: 20Gi - # resources: {} - # operator: - # resources: {} - # alertmanager: - # replicas: 1 - # resources: {} - # notification_manager: - # resources: {} - # operator: - # resources: {} - # proxy: - # resources: {} - gpu: - nvidia_dcgm_exporter: - enabled: false - # resources: {} - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - istio: - components: - ingressGateways: - - name: istio-ingressgateway - enabled: false - cni: - enabled: false - edgeruntime: - enabled: false - kubeedge: - enabled: false - cloudCore: - cloudHub: - advertiseAddress: - - "" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - # resources: {} - # hostNetWork: false - iptables-manager: - enabled: true - mode: "external" - # resources: {} - # edgeService: - # resources: {} - gatekeeper: - enabled: false - # controller_manager: - # resources: {} - # audit: - # resources: {} - terminal: - timeout: 600 -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/cc_v341.go b/cmd/kk/pkg/version/kubesphere/templates/cc_v341.go deleted file mode 100644 index fd2ec6c0..00000000 --- a/cmd/kk/pkg/version/kubesphere/templates/cc_v341.go +++ /dev/null @@ -1,229 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var V341 = template.Must(template.New("v3.4.1").Parse( - dedent.Dedent(` ---- -apiVersion: installer.kubesphere.io/v1alpha1 -kind: ClusterConfiguration -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - version: {{ .Tag }} -spec: - persistence: - storageClass: "" - authentication: - jwtSecret: "" - local_registry: "" - # dev_tag: "" - etcd: - monitoring: false - endpointIps: localhost - port: 2379 - tlsEnable: true - common: - core: - console: - enableMultiLogin: true - port: 30880 - type: NodePort - # apiserver: - # resources: {} - # controllerManager: - # resources: {} - redis: - enabled: false - enableHA: false - volumeSize: 2Gi - openldap: - enabled: false - volumeSize: 2Gi - minio: - volumeSize: 20Gi - monitoring: - # type: external - endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 - GPUMonitoring: - enabled: false - gpu: - kinds: - - resourceName: "nvidia.com/gpu" - resourceType: "GPU" - default: true - es: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - enabled: false - logMaxAge: 7 - elkPrefix: logstash - basicAuth: - enabled: false - username: "" - password: "" - externalElasticsearchHost: "" - externalElasticsearchPort: "" - opensearch: - # master: - # volumeSize: 4Gi - # replicas: 1 - # resources: {} - # data: - # volumeSize: 20Gi - # replicas: 1 - # resources: {} - enabled: true - logMaxAge: 7 - opensearchPrefix: whizard - basicAuth: - enabled: true - username: "admin" - password: "admin" - externalOpensearchHost: "" - externalOpensearchPort: "" - dashboard: - enabled: false - alerting: - enabled: false - # thanosruler: - # replicas: 1 - # resources: {} - auditing: - enabled: false - # operator: - # resources: {} - # webhook: - # resources: {} - devops: - enabled: false - jenkinsCpuReq: 0.5 - jenkinsCpuLim: 1 - jenkinsMemoryReq: 4Gi - jenkinsMemoryLim: 4Gi - jenkinsVolumeSize: 16Gi - events: - enabled: false - # operator: - # resources: {} - # exporter: - # resources: {} - ruler: - enabled: true - replicas: 2 - # resources: {} - logging: - enabled: false - logsidecar: - enabled: true - replicas: 2 - # resources: {} - metrics_server: - enabled: false - monitoring: - storageClass: "" - node_exporter: - port: 9100 - # resources: {} - # kube_rbac_proxy: - # resources: {} - # kube_state_metrics: - # resources: {} - # prometheus: - # replicas: 1 - # volumeSize: 20Gi - # resources: {} - # operator: - # resources: {} - # alertmanager: - # replicas: 1 - # resources: {} - # notification_manager: - # resources: {} - # operator: - # resources: {} - # proxy: - # resources: {} - gpu: - nvidia_dcgm_exporter: - enabled: false - # resources: {} - multicluster: - clusterRole: none - network: - networkpolicy: - enabled: false - ippool: - type: none - topology: - type: none - openpitrix: - store: - enabled: false - servicemesh: - enabled: false - istio: - components: - ingressGateways: - - name: istio-ingressgateway - enabled: false - cni: - enabled: false - edgeruntime: - enabled: false - kubeedge: - enabled: false - cloudCore: - cloudHub: - advertiseAddress: - - "" - service: - cloudhubNodePort: "30000" - cloudhubQuicNodePort: "30001" - cloudhubHttpsNodePort: "30002" - cloudstreamNodePort: "30003" - tunnelNodePort: "30004" - # resources: {} - # hostNetWork: false - iptables-manager: - enabled: true - mode: "external" - # resources: {} - # edgeService: - # resources: {} - gatekeeper: - enabled: false - # controller_manager: - # resources: {} - # audit: - # resources: {} - terminal: - timeout: 600 -`))) diff --git a/cmd/kk/pkg/version/kubesphere/templates/installer.go b/cmd/kk/pkg/version/kubesphere/templates/installer.go deleted file mode 100644 index 943b3ae6..00000000 --- a/cmd/kk/pkg/version/kubesphere/templates/installer.go +++ /dev/null @@ -1,336 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package templates - -import ( - "text/template" - - "github.com/lithammer/dedent" -) - -var ( - KsInstaller = template.Must(template.New("kubesphere.yaml").Parse( - dedent.Dedent(` ---- -apiVersion: v1 -kind: Namespace -metadata: - name: kubesphere-system - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ks-installer - namespace: kubesphere-system - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: clusterconfigurations.installer.kubesphere.io -spec: - group: installer.kubesphere.io - versions: - - name: v1alpha1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - x-kubernetes-preserve-unknown-fields: true - status: - type: object - x-kubernetes-preserve-unknown-fields: true - scope: Namespaced - names: - plural: clusterconfigurations - singular: clusterconfiguration - kind: ClusterConfiguration - shortNames: - - cc - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: ks-installer -rules: -- apiGroups: - - "" - resources: - - '*' - verbs: - - '*' -- apiGroups: - - apps - resources: - - '*' - verbs: - - '*' -- apiGroups: - - extensions - resources: - - '*' - verbs: - - '*' -- apiGroups: - - batch - resources: - - '*' - verbs: - - '*' -- apiGroups: - - rbac.authorization.k8s.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - apiregistration.k8s.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - apiextensions.k8s.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - tenant.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - certificates.k8s.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - devops.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - monitoring.coreos.com - resources: - - '*' - verbs: - - '*' -- apiGroups: - - logging.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - jaegertracing.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - storage.k8s.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - admissionregistration.k8s.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - policy - resources: - - '*' - verbs: - - '*' -- apiGroups: - - autoscaling - resources: - - '*' - verbs: - - '*' -- apiGroups: - - networking.istio.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - config.istio.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - iam.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - notification.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - auditing.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - events.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - core.kubefed.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - installer.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - storage.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - security.istio.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - monitoring.kiali.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - kiali.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - networking.k8s.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - kubeedge.kubesphere.io - - edgeruntime.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - types.kubefed.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - monitoring.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - application.kubesphere.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - alerting.kubesphere.io - resources: - - '*' - verbs: - - '*' - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: ks-installer -subjects: -- kind: ServiceAccount - name: ks-installer - namespace: kubesphere-system -roleRef: - kind: ClusterRole - name: ks-installer - apiGroup: rbac.authorization.k8s.io - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ks-installer - namespace: kubesphere-system - labels: - app: ks-installer - version: {{ .Tag }} -spec: - replicas: 1 - selector: - matchLabels: - app: ks-installer - template: - metadata: - labels: - app: ks-installer - spec: - serviceAccountName: ks-installer - containers: - - name: installer - image: {{ .Repo }}/ks-installer:{{ .Tag }} - imagePullPolicy: IfNotPresent - volumeMounts: - - mountPath: /etc/localtime - name: host-time - volumes: - - hostPath: - path: /etc/localtime - type: "" - name: host-time - - `))) -) diff --git a/cmd/kk/pkg/version/kubesphere/version_enum.go b/cmd/kk/pkg/version/kubesphere/version_enum.go deleted file mode 100644 index d29c71a0..00000000 --- a/cmd/kk/pkg/version/kubesphere/version_enum.go +++ /dev/null @@ -1,176 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubesphere - -import ( - "fmt" - "strings" - - versionutil "k8s.io/apimachinery/pkg/util/version" -) - -type Version int - -const ( - V211 Version = iota - V300 - V310 - V311 - V320 - V321 - V330 - V331 - V332 - V340 - V341 -) - -var VersionList = []Version{ - V211, - V300, - V310, - V311, - V320, - V321, - V330, - V331, - V332, - V340, - V341, -} - -var VersionMap = map[string]*KsInstaller{ - V211.String(): KsV211, - V300.String(): KsV300, - V310.String(): KsV310, - V311.String(): KsV311, - V320.String(): KsV320, - V321.String(): KsV321, - V330.String(): KsV330, - V331.String(): KsV331, - V332.String(): KsV332, - V340.String(): KsV340, - V341.String(): KsV341, -} - -var CNSource = map[string]bool{ - V310.String(): true, - V311.String(): true, - V320.String(): true, - V321.String(): true, - V330.String(): true, - V331.String(): true, - V332.String(): true, - V340.String(): true, - V341.String(): true, -} - -func (v Version) String() string { - switch v { - case V211: - return "v2.1.1" - case V300: - return "v3.0.0" - case V310: - return "v3.1.0" - case V311: - return "v3.1.1" - case V320: - return "v3.2.0" - case V321: - return "v3.2.1" - case V330: - return "v3.3.0" - case V331: - return "v3.3.1" - case V332: - return "v3.3.2" - case V340: - return "v3.4.0" - case V341: - return "v3.4.1" - default: - return "invalid option" - } -} - -func VersionsStringArr() []string { - strArr := make([]string, 0, len(VersionList)) - for i, v := range VersionList { - strArr[i] = v.String() - } - return strArr -} - -func StabledVersionSupport(version string) (*KsInstaller, bool) { - if ks, ok := VersionMap[version]; ok { - return ks, true - } - return nil, false -} - -func LatestRelease(version string) (*KsInstaller, bool) { - if strings.HasPrefix(version, "nightly-") || - version == "latest" || - version == "master" || - strings.Contains(version, "release") { - return Latest(), true - } - - v, err := versionutil.ParseGeneric(version) - if err != nil { - return nil, false - } - - str := fmt.Sprintf("v%s", v.String()) - if ks, ok := StabledVersionSupport(str); ok { - if ks.Version == Latest().Version { - return ks, true - } - return nil, false - } - - return nil, false -} - -func DevRelease(version string) (*KsInstaller, bool) { - if strings.HasPrefix(version, "nightly-") || - version == "latest" || - version == "master" || - strings.Contains(version, "release") { - return Latest(), true - } - - if _, ok := StabledVersionSupport(version); ok { - return nil, false - } - - v, err := versionutil.ParseGeneric(version) - if err != nil { - return nil, false - } - - if ks, ok := StabledVersionSupport(fmt.Sprintf("v%s", v.String())); ok { - return ks, true - } - - return nil, false -} - -func Latest() *KsInstaller { - return VersionMap[VersionList[len(VersionList)-1].String()] -} diff --git a/cmd/kk/pkg/version/kubesphere/version_enum_test.go b/cmd/kk/pkg/version/kubesphere/version_enum_test.go deleted file mode 100644 index 5c7710b0..00000000 --- a/cmd/kk/pkg/version/kubesphere/version_enum_test.go +++ /dev/null @@ -1,242 +0,0 @@ -/* - Copyright 2021 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubesphere - -import ( - "reflect" - "testing" -) - -func TestDevRelease(t *testing.T) { - tests := []struct { - name string - version string - want *KsInstaller - ok bool - }{ - { - name: "test_v3.2.1-rc.1", - version: "v3.2.1-rc.1", - want: KsV321, - ok: true, - }, - { - name: "test_v3.2.1", - version: "v3.2.1", - want: nil, - ok: false, - }, - { - name: "test_v3.2.0", - version: "v3.2.0", - want: nil, - ok: false, - }, - { - name: "test_v3.2.0-alpha.1", - version: "v3.2.0-alpha.1", - want: KsV320, - ok: true, - }, - { - name: "test_v3.2.0-beta.1", - version: "v3.2.0-beta.1", - want: KsV320, - ok: true, - }, - { - name: "test_v3.1.0-alpha.1", - version: "v3.1.0-alpha.1", - want: KsV310, - ok: true, - }, - { - name: "test_latest", - version: "latest", - want: KsV331, - ok: true, - }, - { - name: "test_master", - version: "master", - want: KsV331, - ok: true, - }, - { - name: "test_release-3.3", - version: "release-3.3", - want: KsV331, - ok: true, - }, - { - name: "test_v1.2.0", - version: "v1.2.0", - want: nil, - ok: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, ok := DevRelease(tt.version) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("DevRelease() got = %v, want %v", got, tt.want) - } - if ok != tt.ok { - t.Errorf("DevRelease() got1 = %v, want %v", ok, tt.ok) - } - }) - } -} - -func TestLatest(t *testing.T) { - tests := []struct { - name string - want *KsInstaller - }{ - { - name: "test_latest", - want: KsV331, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := Latest(); !reflect.DeepEqual(got, tt.want) { - t.Errorf("Latest() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestLatestRelease(t *testing.T) { - tests := []struct { - name string - version string - want *KsInstaller - ok bool - }{ - { - name: "test_latest", - version: "latest", - want: KsV331, - ok: true, - }, - { - name: "test_master", - version: "master", - want: KsV331, - ok: true, - }, - { - name: "test_release-3.2", - version: "release-3.2", - want: KsV331, - ok: true, - }, - { - name: "test_v3.3.1", - version: "v3.3.1", - want: KsV331, - ok: true, - }, - { - name: "test_v3.3.1-rc.1", - version: "v3.3.1-rc.1", - want: KsV331, - ok: true, - }, - { - name: "test_v3.2.0", - version: "v3.2.0", - want: nil, - ok: false, - }, - { - name: "test_v3.1.0", - version: "v3.1.0", - want: nil, - ok: false, - }, - { - name: "test_v3.2.0-alpha.1", - version: "v3.2.0-alpha.1", - want: nil, - ok: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, ok := LatestRelease(tt.version) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("LatestRelease() got = %v, want %v", got, tt.want) - } - if ok != tt.ok { - t.Errorf("LatestRelease() got1 = %v, want %v", ok, tt.ok) - } - }) - } -} - -func TestStabledVersionSupport(t *testing.T) { - tests := []struct { - name string - version string - want *KsInstaller - ok bool - }{ - { - name: "test_v3.2.1-rc.1", - version: "v3.2.1-rc.1", - want: nil, - ok: false, - }, - { - name: "test_v3.2.0", - version: "v3.2.0", - want: KsV320, - ok: true, - }, - { - name: "test_3.2.0", - version: "3.2.0", - want: nil, - ok: false, - }, - { - name: "test_v3.2.0-alpha.1", - version: "v3.2.0-alpha.1", - want: nil, - ok: false, - }, - { - name: "test_v1.2.0", - version: "v1.2.0", - want: nil, - ok: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, ok := StabledVersionSupport(tt.version) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("StabledVersionSupport() got = %v, want %v", got, tt.want) - } - if ok != tt.ok { - t.Errorf("StabledVersionSupport() got1 = %v, want %v", ok, tt.ok) - } - }) - } -} diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml deleted file mode 100644 index 73a14e14..00000000 --- a/config/certmanager/certificate.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize - dnsNames: - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: $(SERVICE_NAME)-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml deleted file mode 100644 index bebea5a5..00000000 --- a/config/certmanager/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -resources: -- certificate.yaml - -configurations: -- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml deleted file mode 100644 index d6109c9d..00000000 --- a/config/certmanager/kustomizeconfig.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# This configuration is for teaching kustomize how to update name ref and var substitution -nameReference: -- kind: Issuer - group: cert-manager.io - fieldSpecs: - - kind: Certificate - group: cert-manager.io - path: spec/issuerRef/name - -varReference: -- kind: Certificate - group: cert-manager.io - path: spec/commonName -- kind: Certificate - group: cert-manager.io - path: spec/dnsNames -- kind: Certificate - group: cert-manager.io - path: spec/secretName diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_kkclusters.yaml deleted file mode 100644 index ba2bf021..00000000 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkclusters.yaml +++ /dev/null @@ -1,409 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.1 - creationTimestamp: null - name: kkclusters.infrastructure.cluster.x-k8s.io -spec: - group: infrastructure.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: KKCluster - listKind: KKClusterList - plural: kkclusters - shortNames: - - kkc - singular: kkcluster - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Cluster to which this KKClusters belongs - jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name - name: Cluster - type: string - - description: Cluster infrastructure is ready for SSH instances - jsonPath: .status.ready - name: Ready - type: string - - description: API Endpoint - jsonPath: .spec.controlPlaneEndpoint - name: Endpoint - priority: 1 - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: KKCluster is the Schema for the kkclusters API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KKClusterSpec defines the desired state of KKCluster - properties: - component: - description: Component is optional configuration for modifying the - FTP server that gets the necessary components for the cluster. - properties: - host: - description: Host is the host to download the binaries. - type: string - overrides: - description: Overrides is a list of components download information - that need to be overridden. - items: - description: Override is a component download information that - need to be overridden. - properties: - arch: - description: Arch is the component arch. e.g. amd64, arm64, - etc. - type: string - checksum: - description: Checksum is the SHA256 checksum of the binary. - properties: - path: - description: Path defines the URL path, which is the - path of the checksum file. - type: string - value: - description: Value is the checksum string value. - type: string - type: object - id: - description: ID is the component id name. e.g. kubeadm, - kubelet, containerd, etc. - type: string - path: - description: Path defines the URL path, which is the string - of information that comes after the top level domain name. - type: string - url: - description: URL is the download url of the binaries. - type: string - version: - description: Version is the component version. e.g. v1.21.1, - v1.22.0, etc. - type: string - type: object - type: array - zone: - description: 'ZONE is the zone of the KKCluster where can get - the binaries. If you have problem to access https://storage.googleapis.com, - you can set "zone: cn".' - type: string - type: object - controlPlaneEndpoint: - description: ControlPlaneEndpoint represents the endpoint used to - communicate with the control plane. - properties: - host: - description: The hostname on which the API server is serving. - type: string - port: - description: The port on which the API server is serving. - format: int32 - type: integer - required: - - host - - port - type: object - controlPlaneLoadBalancer: - description: ControlPlaneLoadBalancer is optional configuration for - customizing control plane behavior. - properties: - host: - description: The hostname on which the API server is serving. - type: string - type: object - distribution: - description: Distribution represents the Kubernetes distribution type - of the cluster. - type: string - nodes: - description: Nodes represents the information about the nodes available - to the cluster - properties: - auth: - description: Auth is the SSH authentication information of all - instance. It is a global auth configuration. - properties: - password: - description: Password is the password for SSH authentication. - type: string - port: - description: Port is the port for SSH authentication. - type: integer - privateKey: - description: PrivateKey is the value of the private key for - SSH authentication. - type: string - privateKeyPath: - description: PrivateKeyFile is the path to the private key - for SSH authentication. - type: string - secret: - description: Secret is the secret of the PrivateKey or Password - for SSH authentication.It should in the same namespace as - capkk. When Password is empty, replace it with data.password. - When PrivateKey is empty, replace it with data.privateKey - type: string - timeout: - description: Timeout is the timeout for establish an SSH connection. - format: int64 - type: integer - user: - description: User is the username for SSH authentication. - type: string - type: object - instances: - description: Instances defines all instance contained in kkcluster. - items: - description: InstanceInfo defines the information about the - instance. - properties: - address: - description: Address is the IP address of the machine. - type: string - arch: - description: Arch is the architecture of the machine. e.g. - "amd64", "arm64". - type: string - auth: - description: Auth is the SSH authentication information - of this machine. It will override the global auth configuration. - properties: - password: - description: Password is the password for SSH authentication. - type: string - port: - description: Port is the port for SSH authentication. - type: integer - privateKey: - description: PrivateKey is the value of the private - key for SSH authentication. - type: string - privateKeyPath: - description: PrivateKeyFile is the path to the private - key for SSH authentication. - type: string - secret: - description: Secret is the secret of the PrivateKey - or Password for SSH authentication.It should in the - same namespace as capkk. When Password is empty, replace - it with data.password. When PrivateKey is empty, replace - it with data.privateKey - type: string - timeout: - description: Timeout is the timeout for establish an - SSH connection. - format: int64 - type: integer - user: - description: User is the username for SSH authentication. - type: string - type: object - internalAddress: - description: InternalAddress is the internal IP address - of the machine. - type: string - name: - description: Name is the host name of the machine. - minLength: 1 - type: string - roles: - description: Roles is the role of the machine. - items: - description: Role represents a role of a node. - type: string - type: array - type: object - type: array - required: - - instances - type: object - registry: - description: Registry represents the cluster image registry used to - pull the images. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this - representation of an object. Servers should convert recognized - schemas to the latest internal value, and may reject unrecognized - values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - auth: - description: Auth defines the auth of this PrivateRegistry. - properties: - caFile: - description: CAFile is an SSL Certificate Authority file used - to secure etcd communication. - type: string - certFile: - description: CertFile is an SSL certification file used to - secure etcd communication. - type: string - certsPath: - description: CertsPath defines the path of the certs files - of this PrivateRegistry. - type: string - insecureSkipVerify: - description: InsecureSkipVerify allow contacting this PrivateRegistry - over HTTPS with failed TLS verification. - type: boolean - keyFile: - description: KeyFile is an SSL key file used to secure etcd - communication. - type: string - password: - description: Password defines the password of this PrivateRegistry. - type: string - plainHTTP: - description: PlainHTTP allow contacting this PrivateRegistry - over HTTP. - type: boolean - username: - description: Username defines the username of this PrivateRegistry. - type: string - required: - - certsPath - - insecureSkipVerify - - password - - plainHTTP - - username - type: object - insecureRegistries: - description: InsecureRegistries defines the insecure registries - of ContainerManager. - items: - type: string - type: array - kind: - description: 'Kind is a string value representing the REST resource - this object represents. Servers may infer this from the endpoint - the client submits requests to. Cannot be updated. In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - namespaceOverride: - description: NamespaceOverride defines the namespace override - of this PrivateRegistry. - type: string - privateRegistry: - description: PrivateRegistry defines the private registry address - of ContainerManager. - type: string - registryMirrors: - description: RegistryMirrors defines the registry mirrors of this - PrivateRegistry. - items: - type: string - type: array - required: - - auth - - namespaceOverride - - privateRegistry - type: object - required: - - nodes - type: object - status: - description: KKClusterStatus defines the observed state of KKCluster - properties: - conditions: - description: Conditions defines current service state of the KKMachine. - items: - description: Condition defines an observation of a Cluster API resource - operational state. - properties: - lastTransitionTime: - description: Last time the condition transitioned from one status - to another. This should be when the underlying condition changed. - If that is not known, then using the time when the API field - changed is acceptable. - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. This field may be empty. - type: string - reason: - description: The reason for the condition's last transition - in CamelCase. The specific API may choose whether or not this - field is considered a guaranteed API. This field may not be - empty. - type: string - severity: - description: Severity provides an explicit classification of - Reason code, so the users or machines can immediately understand - the current situation and act accordingly. The Severity field - MUST be set only when Status=False. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of condition in CamelCase or in foo.example.com/CamelCase. - Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. - type: string - required: - - lastTransitionTime - - status - - type - type: object - type: array - failureMessage: - description: "FailureMessage will be set in the event that there is - a terminal problem reconciling the Machine and will contain a more - verbose string suitable for logging and human consumption. \n This - field should not be set for transitive errors that a controller - faces that are expected to be fixed automatically over time (like - service outages), but instead indicate that something is fundamentally - wrong with the Machine's spec or the configuration of the controller, - and that manual intervention is required. Examples of terminal errors - would be invalid combinations of settings in the spec, values that - are unsupported by the controller, or the responsible controller - itself being critically misconfigured. \n Any transient errors that - occur during the reconciliation of Machines can be added as events - to the Machine object and/or logged in the controller's output." - type: string - failureReason: - description: "FailureReason will be set in the event that there is - a terminal problem reconciling the Machine and will contain a succinct - value suitable for machine interpretation. \n This field should - not be set for transitive errors that a controller faces that are - expected to be fixed automatically over time (like service outages), - but instead indicate that something is fundamentally wrong with - the Machine's spec or the configuration of the controller, and that - manual intervention is required. Examples of terminal errors would - be invalid combinations of settings in the spec, values that are - unsupported by the controller, or the responsible controller itself - being critically misconfigured. \n Any transient errors that occur - during the reconciliation of Machines can be added as events to - the Machine object and/or logged in the controller's output." - type: string - ready: - default: false - type: boolean - required: - - ready - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_kkclustertemplates.yaml deleted file mode 100644 index 098b2aaf..00000000 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkclustertemplates.yaml +++ /dev/null @@ -1,358 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.1 - creationTimestamp: null - name: kkclustertemplates.infrastructure.cluster.x-k8s.io -spec: - group: infrastructure.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: KKClusterTemplate - listKind: KKClusterTemplateList - plural: kkclustertemplates - shortNames: - - kkct - singular: kkclustertemplate - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Time duration since creation of KKClusterTemplate - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: KKClusterTemplate is the Schema for the kkclustertemplates API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KKClusterTemplateSpec defines the desired state of KKClusterTemplate - properties: - template: - description: KKClusterTemplateResource Standard object's metadata - properties: - metadata: - description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' - properties: - annotations: - additionalProperties: - type: string - description: 'Annotations is an unstructured key value map - stored with a resource that may be set by external tools - to store and retrieve arbitrary metadata. They are not queryable - and should be preserved when modifying objects. More info: - http://kubernetes.io/docs/user-guide/annotations' - type: object - labels: - additionalProperties: - type: string - description: 'Map of string keys and values that can be used - to organize and categorize (scope and select) objects. May - match selectors of replication controllers and services. - More info: http://kubernetes.io/docs/user-guide/labels' - type: object - type: object - spec: - description: KKClusterSpec defines the desired state of KKCluster - properties: - component: - description: Component is optional configuration for modifying - the FTP server that gets the necessary components for the - cluster. - properties: - host: - description: Host is the host to download the binaries. - type: string - overrides: - description: Overrides is a list of components download - information that need to be overridden. - items: - description: Override is a component download information - that need to be overridden. - properties: - arch: - description: Arch is the component arch. e.g. amd64, - arm64, etc. - type: string - checksum: - description: Checksum is the SHA256 checksum of - the binary. - properties: - path: - description: Path defines the URL path, which - is the path of the checksum file. - type: string - value: - description: Value is the checksum string value. - type: string - type: object - id: - description: ID is the component id name. e.g. kubeadm, - kubelet, containerd, etc. - type: string - path: - description: Path defines the URL path, which is - the string of information that comes after the - top level domain name. - type: string - url: - description: URL is the download url of the binaries. - type: string - version: - description: Version is the component version. e.g. - v1.21.1, v1.22.0, etc. - type: string - type: object - type: array - zone: - description: 'ZONE is the zone of the KKCluster where - can get the binaries. If you have problem to access - https://storage.googleapis.com, you can set "zone: cn".' - type: string - type: object - controlPlaneEndpoint: - description: ControlPlaneEndpoint represents the endpoint - used to communicate with the control plane. - properties: - host: - description: The hostname on which the API server is serving. - type: string - port: - description: The port on which the API server is serving. - format: int32 - type: integer - required: - - host - - port - type: object - controlPlaneLoadBalancer: - description: ControlPlaneLoadBalancer is optional configuration - for customizing control plane behavior. - properties: - host: - description: The hostname on which the API server is serving. - type: string - type: object - distribution: - description: Distribution represents the Kubernetes distribution - type of the cluster. - type: string - nodes: - description: Nodes represents the information about the nodes - available to the cluster - properties: - auth: - description: Auth is the SSH authentication information - of all instance. It is a global auth configuration. - properties: - password: - description: Password is the password for SSH authentication. - type: string - port: - description: Port is the port for SSH authentication. - type: integer - privateKey: - description: PrivateKey is the value of the private - key for SSH authentication. - type: string - privateKeyPath: - description: PrivateKeyFile is the path to the private - key for SSH authentication. - type: string - secret: - description: Secret is the secret of the PrivateKey - or Password for SSH authentication.It should in - the same namespace as capkk. When Password is empty, - replace it with data.password. When PrivateKey is - empty, replace it with data.privateKey - type: string - timeout: - description: Timeout is the timeout for establish - an SSH connection. - format: int64 - type: integer - user: - description: User is the username for SSH authentication. - type: string - type: object - instances: - description: Instances defines all instance contained - in kkcluster. - items: - description: InstanceInfo defines the information about - the instance. - properties: - address: - description: Address is the IP address of the machine. - type: string - arch: - description: Arch is the architecture of the machine. - e.g. "amd64", "arm64". - type: string - auth: - description: Auth is the SSH authentication information - of this machine. It will override the global auth - configuration. - properties: - password: - description: Password is the password for SSH - authentication. - type: string - port: - description: Port is the port for SSH authentication. - type: integer - privateKey: - description: PrivateKey is the value of the - private key for SSH authentication. - type: string - privateKeyPath: - description: PrivateKeyFile is the path to the - private key for SSH authentication. - type: string - secret: - description: Secret is the secret of the PrivateKey - or Password for SSH authentication.It should - in the same namespace as capkk. When Password - is empty, replace it with data.password. When - PrivateKey is empty, replace it with data.privateKey - type: string - timeout: - description: Timeout is the timeout for establish - an SSH connection. - format: int64 - type: integer - user: - description: User is the username for SSH authentication. - type: string - type: object - internalAddress: - description: InternalAddress is the internal IP - address of the machine. - type: string - name: - description: Name is the host name of the machine. - minLength: 1 - type: string - roles: - description: Roles is the role of the machine. - items: - description: Role represents a role of a node. - type: string - type: array - type: object - type: array - required: - - instances - type: object - registry: - description: Registry represents the cluster image registry - used to pull the images. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema - of this representation of an object. Servers should - convert recognized schemas to the latest internal value, - and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - auth: - description: Auth defines the auth of this PrivateRegistry. - properties: - caFile: - description: CAFile is an SSL Certificate Authority - file used to secure etcd communication. - type: string - certFile: - description: CertFile is an SSL certification file - used to secure etcd communication. - type: string - certsPath: - description: CertsPath defines the path of the certs - files of this PrivateRegistry. - type: string - insecureSkipVerify: - description: InsecureSkipVerify allow contacting this - PrivateRegistry over HTTPS with failed TLS verification. - type: boolean - keyFile: - description: KeyFile is an SSL key file used to secure - etcd communication. - type: string - password: - description: Password defines the password of this - PrivateRegistry. - type: string - plainHTTP: - description: PlainHTTP allow contacting this PrivateRegistry - over HTTP. - type: boolean - username: - description: Username defines the username of this - PrivateRegistry. - type: string - required: - - certsPath - - insecureSkipVerify - - password - - plainHTTP - - username - type: object - insecureRegistries: - description: InsecureRegistries defines the insecure registries - of ContainerManager. - items: - type: string - type: array - kind: - description: 'Kind is a string value representing the - REST resource this object represents. Servers may infer - this from the endpoint the client submits requests to. - Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - namespaceOverride: - description: NamespaceOverride defines the namespace override - of this PrivateRegistry. - type: string - privateRegistry: - description: PrivateRegistry defines the private registry - address of ContainerManager. - type: string - registryMirrors: - description: RegistryMirrors defines the registry mirrors - of this PrivateRegistry. - items: - type: string - type: array - required: - - auth - - namespaceOverride - - privateRegistry - type: object - required: - - nodes - type: object - required: - - spec - type: object - required: - - template - type: object - type: object - served: true - storage: true - subresources: {} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkinstances.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_kkinstances.yaml deleted file mode 100644 index 1f6285b2..00000000 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkinstances.yaml +++ /dev/null @@ -1,336 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.1 - creationTimestamp: null - name: kkinstances.infrastructure.cluster.x-k8s.io -spec: - group: infrastructure.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: KKInstance - listKind: KKInstanceList - plural: kkinstances - shortNames: - - kki - singular: kkinstance - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: kubekey instance hostname - jsonPath: .spec.name - name: Hostname - type: string - - description: kubekey instance address - jsonPath: .spec.address - name: Address - type: string - - description: KKInstance state - jsonPath: .status.instanceState - name: State - type: string - - description: Time duration since creation of KKInstance - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - description: Kubernetes version associated with this KKInstance - jsonPath: .status.nodeInfo.kubeletVersion - name: Version - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: KKInstance is the Schema for the kkinstances API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KKInstanceSpec defines the desired state of KKInstance - properties: - address: - description: Address is the IP address of the machine. - type: string - arch: - description: Arch is the architecture of the machine. e.g. "amd64", - "arm64". - type: string - auth: - description: Auth is the SSH authentication information of this machine. - It will override the global auth configuration. - properties: - password: - description: Password is the password for SSH authentication. - type: string - port: - description: Port is the port for SSH authentication. - type: integer - privateKey: - description: PrivateKey is the value of the private key for SSH - authentication. - type: string - privateKeyPath: - description: PrivateKeyFile is the path to the private key for - SSH authentication. - type: string - secret: - description: Secret is the secret of the PrivateKey or Password - for SSH authentication.It should in the same namespace as capkk. - When Password is empty, replace it with data.password. When - PrivateKey is empty, replace it with data.privateKey - type: string - timeout: - description: Timeout is the timeout for establish an SSH connection. - format: int64 - type: integer - user: - description: User is the username for SSH authentication. - type: string - type: object - containerManager: - description: ContainerManager is the container manager config of this - machine. - properties: - criDockerdVersion: - description: CRIDockerdVersion defines the version of cri-dockerd, - available only when Type is docker. https://github.com/Mirantis/cri-dockerd - type: string - criSocket: - description: CRISocket is used to connect an existing CRIClient. - type: string - crictlVersion: - description: CRICTLVersion defines the version of CRICTL. - type: string - type: - description: Type defines the type of ContainerManager. "docker", - "containerd" - type: string - version: - description: Version defines the version of ContainerManager. - type: string - type: object - internalAddress: - description: InternalAddress is the internal IP address of the machine. - type: string - name: - description: Name is the host name of the machine. - minLength: 1 - type: string - repository: - description: Repository is the repository config of this machine. - properties: - iso: - description: 'ISO specifies the ISO file name. There are 3 options: - "": empty string means will not install the packages. "none": - no ISO file will be used. And capkk will use the default repository - to install the required packages. "auto": capkk will detect - the ISO file automatically. Only support Ubuntu/Debian/CentOS. - "xxx-20.04-debs-amd64.iso": use the specified name to get the - ISO file name.' - type: string - packages: - description: Packages is a list of packages to be installed. - items: - type: string - type: array - update: - description: Update will update the repository packages list and - cache if it is true. - type: boolean - type: object - roles: - description: Roles is the role of the machine. - items: - description: Role represents a role of a node. - type: string - type: array - type: object - status: - description: KKInstanceStatus defines the observed state of KKInstance - properties: - conditions: - description: Conditions defines current service state of the KKMachine. - items: - description: Condition defines an observation of a Cluster API resource - operational state. - properties: - lastTransitionTime: - description: Last time the condition transitioned from one status - to another. This should be when the underlying condition changed. - If that is not known, then using the time when the API field - changed is acceptable. - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. This field may be empty. - type: string - reason: - description: The reason for the condition's last transition - in CamelCase. The specific API may choose whether or not this - field is considered a guaranteed API. This field may not be - empty. - type: string - severity: - description: Severity provides an explicit classification of - Reason code, so the users or machines can immediately understand - the current situation and act accordingly. The Severity field - MUST be set only when Status=False. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of condition in CamelCase or in foo.example.com/CamelCase. - Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. - type: string - required: - - lastTransitionTime - - status - - type - type: object - type: array - failureMessage: - description: "FailureMessage will be set in the event that there is - a terminal problem reconciling the Machine and will contain a more - verbose string suitable for logging and human consumption. \n This - field should not be set for transitive errors that a controller - faces that are expected to be fixed automatically over time (like - service outages), but instead indicate that something is fundamentally - wrong with the Machine's spec or the configuration of the controller, - and that manual intervention is required. Examples of terminal errors - would be invalid combinations of settings in the spec, values that - are unsupported by the controller, or the responsible controller - itself being critically misconfigured. \n Any transient errors that - occur during the reconciliation of Machines can be added as events - to the Machine object and/or logged in the controller's output." - type: string - failureReason: - description: "FailureReason will be set in the event that there is - a terminal problem reconciling the Machine and will contain a succinct - value suitable for machine interpretation. \n This field should - not be set for transitive errors that a controller faces that are - expected to be fixed automatically over time (like service outages), - but instead indicate that something is fundamentally wrong with - the Machine's spec or the configuration of the controller, and that - manual intervention is required. Examples of terminal errors would - be invalid combinations of settings in the spec, values that are - unsupported by the controller, or the responsible controller itself - being critically misconfigured. \n Any transient errors that occur - during the reconciliation of Machines can be added as events to - the Machine object and/or logged in the controller's output." - type: string - instanceState: - description: The current state of the instance. - type: string - nodeInfo: - description: 'NodeInfo is a set of ids/uuids to uniquely identify - the node. More info: https://kubernetes.io/docs/concepts/nodes/node/#info' - properties: - architecture: - description: The Architecture reported by the node - type: string - bootID: - description: Boot ID reported by the node. - type: string - containerRuntimeVersion: - description: ContainerRuntime Version reported by the node through - runtime remote API (e.g. containerd://1.4.2). - type: string - kernelVersion: - description: Kernel Version reported by the node from 'uname -r' - (e.g. 3.16.0-0.bpo.4-amd64). - type: string - kubeProxyVersion: - description: KubeProxy Version reported by the node. - type: string - kubeletVersion: - description: Kubelet Version reported by the node. - type: string - machineID: - description: 'MachineID reported by the node. For unique machine - identification in the cluster this field is preferred. Learn - more from man(5) machine-id: http://man7.org/linux/man-pages/man5/machine-id.5.html' - type: string - operatingSystem: - description: The Operating System reported by the node - type: string - osImage: - description: OS Image reported by the node from /etc/os-release - (e.g. Debian GNU/Linux 7 (wheezy)). - type: string - systemUUID: - description: SystemUUID reported by the node. For unique machine - identification MachineID is preferred. This field is specific - to Red Hat hosts https://access.redhat.com/documentation/en-us/red_hat_subscription_management/1/html/rhsm/uuid - type: string - required: - - architecture - - bootID - - containerRuntimeVersion - - kernelVersion - - kubeProxyVersion - - kubeletVersion - - machineID - - operatingSystem - - osImage - - systemUUID - type: object - nodeRef: - description: NodeRef will point to the corresponding Node if it exists. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - x-kubernetes-map-type: atomic - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachines.yaml deleted file mode 100644 index 9c022c4a..00000000 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachines.yaml +++ /dev/null @@ -1,225 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.1 - creationTimestamp: null - name: kkmachines.infrastructure.cluster.x-k8s.io -spec: - group: infrastructure.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: KKMachine - listKind: KKMachineList - plural: kkmachines - shortNames: - - kkm - singular: kkmachine - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Cluster to which this KKMachine belongs - jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name - name: Cluster - type: string - - description: KKInstance name - jsonPath: .spec.instanceID - name: Instance - type: string - - description: Machine ready status - jsonPath: .status.ready - name: Ready - type: string - - description: Machine object which owns with this KKMachine - jsonPath: .metadata.ownerReferences[?(@.kind=="Machine")].name - name: Machine - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: KKMachine is the Schema for the kkmachines API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KKMachineSpec defines the desired state of KKMachine - properties: - containerManager: - description: ContainerManager is the container manager config of this - machine. - properties: - criDockerdVersion: - description: CRIDockerdVersion defines the version of cri-dockerd, - available only when Type is docker. https://github.com/Mirantis/cri-dockerd - type: string - criSocket: - description: CRISocket is used to connect an existing CRIClient. - type: string - crictlVersion: - description: CRICTLVersion defines the version of CRICTL. - type: string - type: - description: Type defines the type of ContainerManager. "docker", - "containerd" - type: string - version: - description: Version defines the version of ContainerManager. - type: string - type: object - instanceID: - description: InstanceID is the name of the KKInstance. - type: string - providerID: - description: ProviderID is the unique identifier as specified by the - kubekey provider. - type: string - repository: - description: Repository is the repository config of this machine. - properties: - iso: - description: 'ISO specifies the ISO file name. There are 3 options: - "": empty string means will not install the packages. "none": - no ISO file will be used. And capkk will use the default repository - to install the required packages. "auto": capkk will detect - the ISO file automatically. Only support Ubuntu/Debian/CentOS. - "xxx-20.04-debs-amd64.iso": use the specified name to get the - ISO file name.' - type: string - packages: - description: Packages is a list of packages to be installed. - items: - type: string - type: array - update: - description: Update will update the repository packages list and - cache if it is true. - type: boolean - type: object - roles: - description: Roles is the role of the machine. - items: - description: Role represents a role of a node. - type: string - type: array - type: object - status: - description: KKMachineStatus defines the observed state of KKMachine - properties: - addresses: - description: Addresses contains the KK instance associated addresses. - items: - description: MachineAddress contains information for the node's - address. - properties: - address: - description: The machine address. - type: string - type: - description: Machine address type, one of Hostname, ExternalIP - or InternalIP. - type: string - required: - - address - - type - type: object - type: array - conditions: - description: Conditions defines current service state of the KKMachine. - items: - description: Condition defines an observation of a Cluster API resource - operational state. - properties: - lastTransitionTime: - description: Last time the condition transitioned from one status - to another. This should be when the underlying condition changed. - If that is not known, then using the time when the API field - changed is acceptable. - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. This field may be empty. - type: string - reason: - description: The reason for the condition's last transition - in CamelCase. The specific API may choose whether or not this - field is considered a guaranteed API. This field may not be - empty. - type: string - severity: - description: Severity provides an explicit classification of - Reason code, so the users or machines can immediately understand - the current situation and act accordingly. The Severity field - MUST be set only when Status=False. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of condition in CamelCase or in foo.example.com/CamelCase. - Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. - type: string - required: - - lastTransitionTime - - status - - type - type: object - type: array - failureMessage: - description: "FailureMessage will be set in the event that there is - a terminal problem reconciling the Machine and will contain a more - verbose string suitable for logging and human consumption. \n This - field should not be set for transitive errors that a controller - faces that are expected to be fixed automatically over time (like - service outages), but instead indicate that something is fundamentally - wrong with the Machine's spec or the configuration of the controller, - and that manual intervention is required. Examples of terminal errors - would be invalid combinations of settings in the spec, values that - are unsupported by the controller, or the responsible controller - itself being critically misconfigured. \n Any transient errors that - occur during the reconciliation of Machines can be added as events - to the Machine object and/or logged in the controller's output." - type: string - failureReason: - description: "FailureReason will be set in the event that there is - a terminal problem reconciling the Machine and will contain a succinct - value suitable for machine interpretation. \n This field should - not be set for transitive errors that a controller faces that are - expected to be fixed automatically over time (like service outages), - but instead indicate that something is fundamentally wrong with - the Machine's spec or the configuration of the controller, and that - manual intervention is required. Examples of terminal errors would - be invalid combinations of settings in the spec, values that are - unsupported by the controller, or the responsible controller itself - being critically misconfigured. \n Any transient errors that occur - during the reconciliation of Machines can be added as events to - the Machine object and/or logged in the controller's output." - type: string - instanceState: - description: InstanceState is the state of the KK instance for this - machine. - type: string - ready: - description: Ready is true when the provider resource is ready. - type: boolean - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachinetemplates.yaml deleted file mode 100644 index 398a9052..00000000 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_kkmachinetemplates.yaml +++ /dev/null @@ -1,159 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.1 - creationTimestamp: null - name: kkmachinetemplates.infrastructure.cluster.x-k8s.io -spec: - group: infrastructure.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: KKMachineTemplate - listKind: KKMachineTemplateList - plural: kkmachinetemplates - shortNames: - - kkmt - singular: kkmachinetemplate - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Time duration since creation of KKMachineTemplate - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: KKMachineTemplate is the Schema for the kkmachinetemplates API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KKMachineTemplateSpec defines the desired state of KKMachineTemplate - properties: - template: - description: KKMachineTemplateResource describes the data needed to - create am KKMachine from a template. - properties: - metadata: - description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' - properties: - annotations: - additionalProperties: - type: string - description: 'Annotations is an unstructured key value map - stored with a resource that may be set by external tools - to store and retrieve arbitrary metadata. They are not queryable - and should be preserved when modifying objects. More info: - http://kubernetes.io/docs/user-guide/annotations' - type: object - labels: - additionalProperties: - type: string - description: 'Map of string keys and values that can be used - to organize and categorize (scope and select) objects. May - match selectors of replication controllers and services. - More info: http://kubernetes.io/docs/user-guide/labels' - type: object - type: object - spec: - description: Spec is the specification of the desired behavior - of the machine. - properties: - containerManager: - description: ContainerManager is the container manager config - of this machine. - properties: - criDockerdVersion: - description: CRIDockerdVersion defines the version of - cri-dockerd, available only when Type is docker. https://github.com/Mirantis/cri-dockerd - type: string - criSocket: - description: CRISocket is used to connect an existing - CRIClient. - type: string - crictlVersion: - description: CRICTLVersion defines the version of CRICTL. - type: string - type: - description: Type defines the type of ContainerManager. - "docker", "containerd" - type: string - version: - description: Version defines the version of ContainerManager. - type: string - type: object - instanceID: - description: InstanceID is the name of the KKInstance. - type: string - providerID: - description: ProviderID is the unique identifier as specified - by the kubekey provider. - type: string - repository: - description: Repository is the repository config of this machine. - properties: - iso: - description: 'ISO specifies the ISO file name. There are - 3 options: "": empty string means will not install the - packages. "none": no ISO file will be used. And capkk - will use the default repository to install the required - packages. "auto": capkk will detect the ISO file automatically. - Only support Ubuntu/Debian/CentOS. "xxx-20.04-debs-amd64.iso": - use the specified name to get the ISO file name.' - type: string - packages: - description: Packages is a list of packages to be installed. - items: - type: string - type: array - update: - description: Update will update the repository packages - list and cache if it is true. - type: boolean - type: object - roles: - description: Roles is the role of the machine. - items: - description: Role represents a role of a node. - type: string - type: array - type: object - required: - - spec - type: object - required: - - template - type: object - status: - description: KKMachineTemplateStatus defines a status for an KKMachineTemplate. - properties: - capacity: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Capacity defines the resource capacity for this machine. - This value is used for autoscaling from zero operations as defined - in: https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20210310-opt-in-autoscaling-from-zero.md' - type: object - type: object - type: object - served: true - storage: true - subresources: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml deleted file mode 100644 index f56714da..00000000 --- a/config/crd/kustomization.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# This kustomization.yaml is not intended to be run by itself, -# since it depends on service name and namespace that are out of this kustomize package. -# It should be run by config/default -resources: -- bases/infrastructure.cluster.x-k8s.io_kkclusters.yaml -- bases/infrastructure.cluster.x-k8s.io_kkclustertemplates.yaml -- bases/infrastructure.cluster.x-k8s.io_kkmachines.yaml -- bases/infrastructure.cluster.x-k8s.io_kkmachinetemplates.yaml -- bases/infrastructure.cluster.x-k8s.io_kkinstances.yaml -#+kubebuilder:scaffold:crdkustomizeresource - -commonLabels: - cluster.x-k8s.io/v1beta1: v1beta1 - -patchesStrategicMerge: -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. -# patches here are for enabling the conversion webhook for each CRD -- patches/webhook_in_kkclusters.yaml -- patches/webhook_in_kkclustertemplates.yaml -- patches/webhook_in_kkmachines.yaml -- patches/webhook_in_kkmachinetemplates.yaml -- patches/webhook_in_kkinstances.yaml -#+kubebuilder:scaffold:crdkustomizewebhookpatch - -# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. -# patches here are for enabling the CA injection for each CRD -- patches/cainjection_in_kkclusters.yaml -- patches/cainjection_in_kkclustertemplates.yaml -- patches/cainjection_in_kkmachines.yaml -- patches/cainjection_in_kkmachinetemplates.yaml -- patches/cainjection_in_kkinstances.yaml -#+kubebuilder:scaffold:crdkustomizecainjectionpatch - -# the following config is for teaching kustomize how to do kustomization for CRDs. -configurations: -- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml deleted file mode 100644 index ec5c150a..00000000 --- a/config/crd/kustomizeconfig.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# This file is for teaching kustomize how to substitute name and namespace reference in CRD -nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/name - -namespace: -- kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/namespace - create: false - -varReference: -- path: metadata/annotations diff --git a/config/crd/patches/cainjection_in_kkclusters.yaml b/config/crd/patches/cainjection_in_kkclusters.yaml deleted file mode 100644 index aabf0075..00000000 --- a/config/crd/patches/cainjection_in_kkclusters.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: kkclusters.infrastructure.cluster.x-k8s.io diff --git a/config/crd/patches/cainjection_in_kkclustertemplates.yaml b/config/crd/patches/cainjection_in_kkclustertemplates.yaml deleted file mode 100644 index 4f99f807..00000000 --- a/config/crd/patches/cainjection_in_kkclustertemplates.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: kkclustertemplates.infrastructure.cluster.x-k8s.io diff --git a/config/crd/patches/cainjection_in_kkinstances.yaml b/config/crd/patches/cainjection_in_kkinstances.yaml deleted file mode 100644 index e2b9e505..00000000 --- a/config/crd/patches/cainjection_in_kkinstances.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: kkinstances.infrastructure.cluster.x-k8s.io diff --git a/config/crd/patches/cainjection_in_kkmachines.yaml b/config/crd/patches/cainjection_in_kkmachines.yaml deleted file mode 100644 index 30c406ee..00000000 --- a/config/crd/patches/cainjection_in_kkmachines.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: kkmachines.infrastructure.cluster.x-k8s.io diff --git a/config/crd/patches/cainjection_in_kkmachinetemplates.yaml b/config/crd/patches/cainjection_in_kkmachinetemplates.yaml deleted file mode 100644 index 74c7289b..00000000 --- a/config/crd/patches/cainjection_in_kkmachinetemplates.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: kkmachinetemplates.infrastructure.cluster.x-k8s.io diff --git a/config/crd/patches/webhook_in_kkclusters.yaml b/config/crd/patches/webhook_in_kkclusters.yaml deleted file mode 100644 index 3ca82219..00000000 --- a/config/crd/patches/webhook_in_kkclusters.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kkclusters.infrastructure.cluster.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: ["v1", "v1beta1"] - clientConfig: - # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, - # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) - caBundle: Cg== - service: - namespace: system - name: webhook-service - path: /convert diff --git a/config/crd/patches/webhook_in_kkclustertemplates.yaml b/config/crd/patches/webhook_in_kkclustertemplates.yaml deleted file mode 100644 index 466cb622..00000000 --- a/config/crd/patches/webhook_in_kkclustertemplates.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kkclustertemplates.infrastructure.cluster.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: ["v1", "v1beta1"] - clientConfig: - # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, - # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) - caBundle: Cg== - service: - namespace: system - name: webhook-service - path: /convert diff --git a/config/crd/patches/webhook_in_kkinstances.yaml b/config/crd/patches/webhook_in_kkinstances.yaml deleted file mode 100644 index e5be13f1..00000000 --- a/config/crd/patches/webhook_in_kkinstances.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kkinstances.infrastructure.cluster.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: ["v1", "v1beta1"] - clientConfig: - # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, - # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) - caBundle: Cg== - service: - namespace: system - name: webhook-service - path: /convert diff --git a/config/crd/patches/webhook_in_kkmachines.yaml b/config/crd/patches/webhook_in_kkmachines.yaml deleted file mode 100644 index 22058138..00000000 --- a/config/crd/patches/webhook_in_kkmachines.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kkmachines.infrastructure.cluster.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: ["v1", "v1beta1"] - clientConfig: - # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, - # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) - caBundle: Cg== - service: - namespace: system - name: webhook-service - path: /convert diff --git a/config/crd/patches/webhook_in_kkmachinetemplates.yaml b/config/crd/patches/webhook_in_kkmachinetemplates.yaml deleted file mode 100644 index 008ed4a1..00000000 --- a/config/crd/patches/webhook_in_kkmachinetemplates.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kkmachinetemplates.infrastructure.cluster.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: ["v1", "v1beta1"] - clientConfig: - # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, - # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) - caBundle: Cg== - service: - namespace: system - name: webhook-service - path: /convert diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml deleted file mode 100644 index 5e3f4334..00000000 --- a/config/default/kustomization.yaml +++ /dev/null @@ -1,48 +0,0 @@ -namePrefix: capkk- -namespace: capkk-system - -commonLabels: - cluster.x-k8s.io/provider: "infrastructure-kk" - -bases: -- ../rbac -- ../manager -- ../crd -- ../certmanager -- ../webhook - -patchesStrategicMerge: -- manager_webhook_patch.yaml -- webhookcainjection_patch.yaml -- manager_pull_policy.yaml -- manager_image_patch.yaml - -configurations: - - kustomizeconfig.yaml -vars: - - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR - objref: - kind: Certificate - group: cert-manager.io - version: v1 - name: serving-cert # this name should match the one in certificate.yaml - fieldref: - fieldpath: metadata.namespace - - name: CERTIFICATE_NAME - objref: - kind: Certificate - group: cert-manager.io - version: v1 - name: serving-cert # this name should match the one in certificate.yaml - - name: SERVICE_NAMESPACE # namespace of the service - objref: - kind: Service - version: v1 - name: webhook-service - fieldref: - fieldpath: metadata.namespace - - name: SERVICE_NAME - objref: - kind: Service - version: v1 - name: webhook-service diff --git a/config/default/kustomizeconfig.yaml b/config/default/kustomizeconfig.yaml deleted file mode 100644 index 524d39cc..00000000 --- a/config/default/kustomizeconfig.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# This configuration is for teaching kustomize how to update name ref and var substitution -varReference: - - kind: Deployment - path: spec/template/spec/volumes/secret/secretName diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml deleted file mode 100644 index 1cba52d7..00000000 --- a/config/default/manager_image_patch.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - image: docker.io/kubespheredev/capkk-controller:main - name: manager diff --git a/config/default/manager_pull_policy.yaml b/config/default/manager_pull_policy.yaml deleted file mode 100644 index b010a1ee..00000000 --- a/config/default/manager_pull_policy.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - imagePullPolicy: IfNotPresent \ No newline at end of file diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml deleted file mode 100644 index b387eb0e..00000000 --- a/config/default/manager_webhook_patch.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - ports: - - containerPort: 9443 - name: webhook-server - protocol: TCP - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: $(SERVICE_NAME)-cert diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml deleted file mode 100644 index 02ab515d..00000000 --- a/config/default/webhookcainjection_patch.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# This patch add annotation to admission webhook config and -# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: mutating-webhook-configuration - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validating-webhook-configuration - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml deleted file mode 100644 index 7394a6d0..00000000 --- a/config/manager/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: - - manager.yaml diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml deleted file mode 100644 index 36967c89..00000000 --- a/config/manager/manager.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: controller-manager - name: system ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - control-plane: controller-manager -spec: - selector: - matchLabels: - control-plane: controller-manager - replicas: 1 - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: manager - labels: - control-plane: controller-manager - spec: - containers: - - args: - - "--leader-elect" - - "--v=4" - image: controller:latest - name: manager - securityContext: - allowPrivilegeEscalation: false - livenessProbe: - httpGet: - path: /healthz - port: 9440 - initialDelaySeconds: 15 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /readyz - port: 9440 - initialDelaySeconds: 5 - periodSeconds: 10 - serviceAccountName: controller-manager - terminationGracePeriodSeconds: 10 - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml deleted file mode 100644 index d2a86b1a..00000000 --- a/config/manifests/kustomization.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# These resources constitute the fully configured set of manifests -# used to generate the 'manifests/' directory in a bundle. -resources: -- bases/cluster-api-kubekey-provider.clusterserviceversion.yaml -- ../default -- ../samples -- ../scorecard - -# [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix. -# Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager. -# These patches remove the unnecessary "cert" volume and its manager container volumeMount. -#patchesJson6902: -#- target: -# group: apps -# version: v1 -# kind: Deployment -# name: controller-manager -# namespace: system -# patch: |- -# # Remove the manager container's "cert" volumeMount, since OLM will create and mount a set of certs. -# # Update the indices in this path if adding or removing containers/volumeMounts in the manager's Deployment. -# - op: remove -# path: /spec/template/spec/containers/1/volumeMounts/0 -# # Remove the "cert" volume, since OLM will create and mount a set of certs. -# # Update the indices in this path if adding or removing volumes in the manager's Deployment. -# - op: remove -# path: /spec/template/spec/volumes/0 diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml deleted file mode 100644 index ed137168..00000000 --- a/config/prometheus/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- monitor.yaml diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml deleted file mode 100644 index d19136ae..00000000 --- a/config/prometheus/monitor.yaml +++ /dev/null @@ -1,20 +0,0 @@ - -# Prometheus Monitor Service (Metrics) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - control-plane: controller-manager - name: controller-manager-metrics-monitor - namespace: system -spec: - endpoints: - - path: /metrics - port: https - scheme: https - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - tlsConfig: - insecureSkipVerify: true - selector: - matchLabels: - control-plane: controller-manager diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index 51a75db4..00000000 --- a/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 80e1857c..00000000 --- a/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml deleted file mode 100644 index ec7acc0a..00000000 --- a/config/rbac/auth_proxy_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml deleted file mode 100644 index 71f17972..00000000 --- a/config/rbac/auth_proxy_service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - control-plane: controller-manager - name: controller-manager-metrics-service - namespace: system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - selector: - control-plane: controller-manager diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml deleted file mode 100644 index 731832a6..00000000 --- a/config/rbac/kustomization.yaml +++ /dev/null @@ -1,18 +0,0 @@ -resources: -# All RBAC will be applied under this service account in -# the deployment namespace. You may comment out this resource -# if your manager will use a service account that exists at -# runtime. Be sure to update RoleBinding and ClusterRoleBinding -# subjects if changing service account names. -- service_account.yaml -- role.yaml -- role_binding.yaml -- leader_election_role.yaml -- leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml deleted file mode 100644 index 4190ec80..00000000 --- a/config/rbac/leader_election_role.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# permissions to do leader election. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: leader-election-role -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml deleted file mode 100644 index 1d1321ed..00000000 --- a/config/rbac/leader_election_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: leader-election-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: leader-election-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml deleted file mode 100644 index 43298303..00000000 --- a/config/rbac/role.yaml +++ /dev/null @@ -1,162 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: manager-role -rules: -- apiGroups: - - "" - resources: - - configmaps - - events - - secrets - verbs: - - create - - get - - list - - patch - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch -- apiGroups: - - cluster.x-k8s.io - resources: - - '*' - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - cluster.x-k8s.io - resources: - - clusters - - clusters/status - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - cluster.x-k8s.io - resources: - - machinedeployments - - machinedeployments/status - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - cluster.x-k8s.io - resources: - - machines - - machines/status - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - cluster.x-k8s.io - resources: - - machinesets - - machinesets/status - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - controlplane.cluster.x-k8s.io - resources: - - '*' - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - infrastructure.cluster.x-k8s.io - resources: - - '*' - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - infrastructure.cluster.x-k8s.io - resources: - - kkinstances - - kkinstances/finalizers - - kkinstances/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - infrastructure.cluster.x-k8s.io - resources: - - kkmachines - - kkmachines/finalizers - - kkmachines/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml deleted file mode 100644 index 2070ede4..00000000 --- a/config/rbac/role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: manager-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml deleted file mode 100644 index 7cd6025b..00000000 --- a/config/rbac/service_account.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: controller-manager - namespace: system diff --git a/config/samples/hello-kubekey/hello.yaml b/config/samples/hello-kubekey/hello.yaml deleted file mode 100644 index b2b58417..00000000 --- a/config/samples/hello-kubekey/hello.yaml +++ /dev/null @@ -1,188 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: quick-start -spec: - clusterNetwork: - services: - cidrBlocks: [ "10.233.0.0/18" ] - pods: - cidrBlocks: [ "10.233.64.0/18" ] - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: KKCluster - name: quick-start - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: quick-start-control-plane ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKCluster -metadata: - name: quick-start -spec: - component: - zone: "" - nodes: - auth: - user: ubuntu - password: Qcloud@123 - instances: - - name: test1 - address: 192.168.0.3 - - name: test2 - address: 192.168.0.4 - - name: test3 - address: 192.168.0.5 - controlPlaneLoadBalancer: - host: 192.168.0.100 ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: quick-start-control-plane -spec: - replicas: 1 - version: v1.22.10 - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: KKMachineTemplate - name: quick-start - kubeadmConfigSpec: - files: - - content: | - apiVersion: v1 - kind: Pod - metadata: - creationTimestamp: null - name: kube-vip - namespace: kube-system - spec: - containers: - - args: - - manager - env: - - name: address - value: 192.168.0.100 - - name: vip_interface - value: ${VIP_NETWORK_INTERFACE=""} - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: lb_enable - value: "true" - - name: lb_port - value: "6443" - image: ghcr.io/kube-vip/kube-vip:v0.5.0 - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - hostNetwork: true - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - volumes: - - hostPath: - path: /etc/kubernetes/admin.conf - type: FileOrCreate - name: kubeconfig - status: {} - owner: root:root - path: /etc/kubernetes/manifests/kube-vip.yaml - initConfiguration: - nodeRegistration: - criSocket: unix:///var/run/containerd/containerd.sock - joinConfiguration: - nodeRegistration: - criSocket: unix:///var/run/containerd/containerd.sock ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKMachineTemplate -metadata: - name: quick-start - namespace: default -spec: - template: - spec: - roles: - - control-plane - repository: - iso: "none" - update: false ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: "quick-start-md-0" -spec: - clusterName: "quick-start" - replicas: 1 - selector: - matchLabels: - template: - spec: - clusterName: "quick-start" - version: v1.22.10 - bootstrap: - configRef: - name: "quick-start-md-0" - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - infrastructureRef: - name: "quick-start-md-0" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: KKMachineTemplate ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKMachineTemplate -metadata: - name: 'quick-start-md-0' -spec: - template: - spec: - roles: - - worker - repository: - iso: "none" - update: false ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: "quick-start-md-0" -spec: - template: - spec: - joinConfiguration: - nodeRegistration: - criSocket: unix:///var/run/containerd/containerd.sock \ No newline at end of file diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml deleted file mode 100644 index 2bc04a19..00000000 --- a/config/samples/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -## Append samples you want in your CSV to this file as resources ## -resources: -- infrastructure_v1beta1_kkcluster.yaml -- infrastructure_v1beta1_kkclustertemplate.yaml -- infrastructure_v1beta1_kkmachine.yaml -- infrastructure_v1beta1_kkmachinetemplate.yaml -- infrastructure_v1beta1_kkinstance.yaml -#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/scorecard/bases/config.yaml b/config/scorecard/bases/config.yaml deleted file mode 100644 index c7704784..00000000 --- a/config/scorecard/bases/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: scorecard.operatorframework.io/v1alpha3 -kind: Configuration -metadata: - name: config -stages: -- parallel: true - tests: [] diff --git a/config/scorecard/kustomization.yaml b/config/scorecard/kustomization.yaml deleted file mode 100644 index 50cd2d08..00000000 --- a/config/scorecard/kustomization.yaml +++ /dev/null @@ -1,16 +0,0 @@ -resources: -- bases/config.yaml -patchesJson6902: -- path: patches/basic.config.yaml - target: - group: scorecard.operatorframework.io - version: v1alpha3 - kind: Configuration - name: config -- path: patches/olm.config.yaml - target: - group: scorecard.operatorframework.io - version: v1alpha3 - kind: Configuration - name: config -#+kubebuilder:scaffold:patchesJson6902 diff --git a/config/scorecard/patches/basic.config.yaml b/config/scorecard/patches/basic.config.yaml deleted file mode 100644 index 7b4653e8..00000000 --- a/config/scorecard/patches/basic.config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - basic-check-spec - image: quay.io/operator-framework/scorecard-test:v1.20.0 - labels: - suite: basic - test: basic-check-spec-test diff --git a/config/scorecard/patches/olm.config.yaml b/config/scorecard/patches/olm.config.yaml deleted file mode 100644 index 39390f09..00000000 --- a/config/scorecard/patches/olm.config.yaml +++ /dev/null @@ -1,50 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-bundle-validation - image: quay.io/operator-framework/scorecard-test:v1.20.0 - labels: - suite: olm - test: olm-bundle-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-validation - image: quay.io/operator-framework/scorecard-test:v1.20.0 - labels: - suite: olm - test: olm-crds-have-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-resources - image: quay.io/operator-framework/scorecard-test:v1.20.0 - labels: - suite: olm - test: olm-crds-have-resources-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-spec-descriptors - image: quay.io/operator-framework/scorecard-test:v1.20.0 - labels: - suite: olm - test: olm-spec-descriptors-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-status-descriptors - image: quay.io/operator-framework/scorecard-test:v1.20.0 - labels: - suite: olm - test: olm-status-descriptors-test diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml deleted file mode 100644 index 9cf26134..00000000 --- a/config/webhook/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -resources: -- manifests.yaml -- service.yaml - -configurations: -- kustomizeconfig.yaml diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml deleted file mode 100644 index 25e21e3c..00000000 --- a/config/webhook/kustomizeconfig.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# the following config is for teaching kustomize where to look at when substituting vars. -# It requires kustomize v2.1.0 or newer to work properly. -nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: MutatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/name - - kind: ValidatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/name - -namespace: -- kind: MutatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/namespace - create: true -- kind: ValidatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/namespace - create: true - -varReference: -- path: metadata/annotations diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml deleted file mode 100644 index 9757a885..00000000 --- a/config/webhook/manifests.yaml +++ /dev/null @@ -1,214 +0,0 @@ ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - creationTimestamp: null - name: mutating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkcluster - failurePolicy: Fail - name: default.kkcluster.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - kkclusters - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkclustertemplate - failurePolicy: Fail - name: default.kkclustertemplate.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - kkclustertemplates - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkinstance - failurePolicy: Fail - name: default.kkinstance.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - kkinstances - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkmachine - failurePolicy: Fail - name: default.kkmachine.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - kkmachines - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-kkmachinetemplate - failurePolicy: Fail - name: default.kkmachinetemplate.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - kkmachinetemplates - sideEffects: None ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - creationTimestamp: null - name: validating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-kkcluster - failurePolicy: Fail - name: validation.kkcluster.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - kkclusters - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-kkclustertemplate - failurePolicy: Fail - name: validation.kkclustertemplate.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - kkclustertemplates - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-kkinstance - failurePolicy: Fail - name: validation.kkinstance.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - kkinstances - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-kkmachine - failurePolicy: Fail - name: validation.kkmachine.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - kkmachines - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-kkmachinetemplate - failurePolicy: Fail - name: validation.kkmachinetemplate.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - kkmachinetemplates - sideEffects: None diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml deleted file mode 100644 index 9bc95014..00000000 --- a/config/webhook/service.yaml +++ /dev/null @@ -1,10 +0,0 @@ - -apiVersion: v1 -kind: Service -metadata: - name: webhook-service - namespace: system -spec: - ports: - - port: 443 - targetPort: webhook-server diff --git a/controllers/alias.go b/controllers/alias.go deleted file mode 100644 index 4657c3e5..00000000 --- a/controllers/alias.go +++ /dev/null @@ -1,102 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package controllers - -import ( - "context" - "time" - - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/cluster-api/controllers/remote" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - - kkclustercontroller "github.com/kubesphere/kubekey/v3/controllers/kkcluster" - kkinstancecontroller "github.com/kubesphere/kubekey/v3/controllers/kkinstance" - kkmachinecontroller "github.com/kubesphere/kubekey/v3/controllers/kkmachine" -) - -// KKClusterReconciler reconciles a KKCluster object -type KKClusterReconciler struct { - client.Client - Recorder record.EventRecorder - Scheme *runtime.Scheme - WatchFilterValue string - DataDir string -} - -// SetupWithManager sets up the controller with the Manager. -func (r *KKClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { - return (&kkclustercontroller.Reconciler{ - Client: r.Client, - Recorder: r.Recorder, - Scheme: r.Scheme, - WatchFilterValue: r.WatchFilterValue, - DataDir: r.DataDir, - }).SetupWithManager(ctx, mgr, options) -} - -// KKMachineReconciler reconciles a KKMachine object -type KKMachineReconciler struct { - client.Client - Scheme *runtime.Scheme - Recorder record.EventRecorder - Tracker *remote.ClusterCacheTracker - WatchFilterValue string - DataDir string -} - -// SetupWithManager sets up the controller with the Manager. -func (r *KKMachineReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { - return (&kkmachinecontroller.Reconciler{ - Client: r.Client, - Recorder: r.Recorder, - Scheme: r.Scheme, - Tracker: r.Tracker, - WatchFilterValue: r.WatchFilterValue, - DataDir: r.DataDir, - }).SetupWithManager(ctx, mgr, options) -} - -// KKInstanceReconciler reconciles a KKInstance object -type KKInstanceReconciler struct { - client.Client - Scheme *runtime.Scheme - Tracker *remote.ClusterCacheTracker - Recorder record.EventRecorder - WatchFilterValue string - DataDir string - - WaitKKInstanceInterval time.Duration - WaitKKInstanceTimeout time.Duration -} - -// SetupWithManager sets up the controller with the Manager. -func (r *KKInstanceReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { - return (&kkinstancecontroller.Reconciler{ - Client: r.Client, - Recorder: r.Recorder, - Tracker: r.Tracker, - Scheme: r.Scheme, - WatchFilterValue: r.WatchFilterValue, - DataDir: r.DataDir, - WaitKKInstanceInterval: r.WaitKKInstanceInterval, - WaitKKInstanceTimeout: r.WaitKKInstanceTimeout, - }).SetupWithManager(ctx, mgr, options) -} diff --git a/controllers/doc.go b/controllers/doc.go deleted file mode 100644 index 11c669d0..00000000 --- a/controllers/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package controllers implements controllers. -package controllers diff --git a/controllers/kkcluster/doc.go b/controllers/kkcluster/doc.go deleted file mode 100644 index 2c43606e..00000000 --- a/controllers/kkcluster/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package kkcluster implements kkcluster controllers. -package kkcluster diff --git a/controllers/kkcluster/kkcluster_controller.go b/controllers/kkcluster/kkcluster_controller.go deleted file mode 100644 index eae8525a..00000000 --- a/controllers/kkcluster/kkcluster_controller.go +++ /dev/null @@ -1,335 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kkcluster - -import ( - "context" - "fmt" - "net" - "time" - - "github.com/go-logr/logr" - "github.com/google/go-cmp/cmp" - "github.com/pkg/errors" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/cluster-api/util/annotations" - "sigs.k8s.io/cluster-api/util/conditions" - "sigs.k8s.io/cluster-api/util/patch" - "sigs.k8s.io/cluster-api/util/predicates" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" - "github.com/kubesphere/kubekey/v3/pkg/scope" - "github.com/kubesphere/kubekey/v3/util/collections" -) - -const ( - // upgradeCheckFailedRequeueAfter is how long to wait before requeuing a cluster for which the upgrade check failed. - upgradeCheckFailedRequeueAfter = 30 * time.Second -) - -// Reconciler reconciles a KKCluster object -type Reconciler struct { - client.Client - Recorder record.EventRecorder - Scheme *runtime.Scheme - WatchFilterValue string - DataDir string -} - -// SetupWithManager sets up the controller with the Manager. -func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { - log := ctrl.LoggerFrom(ctx) - c, err := ctrl.NewControllerManagedBy(mgr). - WithOptions(options). - For(&infrav1.KKCluster{}). - WithEventFilter(predicates.ResourceHasFilterLabel(log, r.WatchFilterValue)). - WithEventFilter( - predicate.Funcs{ - // Avoid reconciling if the event triggering the reconciliation is related to incremental status updates - // for KKCluster resources only - UpdateFunc: func(e event.UpdateEvent) bool { - if _, ok := e.ObjectOld.(*infrav1.KKCluster); !ok { - return true - } - - oldCluster := e.ObjectOld.(*infrav1.KKCluster).DeepCopy() - newCluster := e.ObjectNew.(*infrav1.KKCluster).DeepCopy() - - oldCluster.Status = infrav1.KKClusterStatus{} - newCluster.Status = infrav1.KKClusterStatus{} - - oldCluster.ObjectMeta.ResourceVersion = "" - newCluster.ObjectMeta.ResourceVersion = "" - - return !cmp.Equal(oldCluster, newCluster) - }, - }, - ). - WithEventFilter(predicates.ResourceIsNotExternallyManaged(log)). - Build(r) - if err != nil { - return errors.Wrap(err, "error creating controller") - } - - return c.Watch( - &source.Kind{Type: &clusterv1.Cluster{}}, - handler.EnqueueRequestsFromMapFunc(r.requeueKKClusterForUnpausedCluster(ctx, log)), - predicates.ClusterUnpaused(log), - ) -} - -// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=*,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status,verbs=get;list;watch;update;patch -// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines;machines/status,verbs=get;list;watch;update;patch -// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=controlplane.cluster.x-k8s.io,resources=*,verbs=get;list;watch;update;patch -// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machinedeployments;machinedeployments/status,verbs=get;list;watch;update;patch -// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machinesets;machinesets/status,verbs=get;list;watch;update;patch -// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=*,verbs=get;list;watch;update;patch -// +kubebuilder:rbac:groups="",resources=secrets;events;configmaps,verbs=get;list;watch;create;patch - -func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, retErr error) { - log := ctrl.LoggerFrom(ctx) - - kkCluster := &infrav1.KKCluster{} - err := r.Get(ctx, req.NamespacedName, kkCluster) - if err != nil { - if apierrors.IsNotFound(err) { - return reconcile.Result{}, nil - } - return reconcile.Result{}, err - } - - // Fetch the Cluster. - cluster, err := util.GetOwnerCluster(ctx, r.Client, kkCluster.ObjectMeta) - if err != nil { - return reconcile.Result{}, err - } - - if cluster == nil { - log.Info("Cluster Controller has not yet set OwnerRef") - return reconcile.Result{}, nil - } - - log = log.WithValues("cluster", cluster.Name) - helper, err := patch.NewHelper(kkCluster, r.Client) - if err != nil { - return reconcile.Result{}, errors.Wrap(err, "failed to init patch helper") - } - - defer func() { - e := helper.Patch( - context.TODO(), - kkCluster, - patch.WithOwnedConditions{Conditions: []clusterv1.ConditionType{ - infrav1.PrincipalPreparedCondition, - }}) - if e != nil { - fmt.Println(e.Error()) - } - }() - - // Create the scope. - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ - Client: r.Client, - Logger: &log, - Cluster: cluster, - KKCluster: kkCluster, - ControllerName: "kkcluster", - RootFsBasePath: r.DataDir, - }) - if err != nil { - return reconcile.Result{}, errors.Errorf("failed to create scope: %+v", err) - } - - // Always close the scope when exiting this function, so we can persist any KKCluster changes. - defer func() { - if err := clusterScope.Close(); err != nil && retErr == nil { - log.Error(err, "failed to patch object") - retErr = err - } - }() - - // Handle deleted clusters - if !kkCluster.DeletionTimestamp.IsZero() { - return r.reconcileDelete(ctx, clusterScope) - } - - // Handle non-deleted clusters - return r.reconcileNormal(ctx, clusterScope) -} - -func (r *Reconciler) reconcileDelete(ctx context.Context, clusterScope *scope.ClusterScope) (ctrl.Result, error) { //nolint:unparam - log := ctrl.LoggerFrom(ctx) - log.V(4).Info("Reconcile KKCluster delete") - - if annotations.IsPaused(clusterScope.Cluster, clusterScope.KKCluster) { - log.Info("KKCluster or linked Cluster is marked as paused. Won't reconcile") - return reconcile.Result{}, nil - } - - // Cluster is deleted so remove the finalizer. - controllerutil.RemoveFinalizer(clusterScope.KKCluster, infrav1.ClusterFinalizer) - return ctrl.Result{}, nil -} - -func (r *Reconciler) reconcileNormal(ctx context.Context, clusterScope *scope.ClusterScope) (reconcile.Result, error) { - log := ctrl.LoggerFrom(ctx) - log.V(4).Info("Reconcile KKCluster normal") - - kkCluster := clusterScope.KKCluster - - // If the KKCluster doesn't have our finalizer, add it. - if controllerutil.AddFinalizer(kkCluster, infrav1.ClusterFinalizer) { - // Register the finalizer immediately to avoid orphaning KK resources on delete - if err := clusterScope.PatchObject(); err != nil { - return reconcile.Result{}, err - } - } - - if _, err := net.LookupIP(kkCluster.Spec.ControlPlaneLoadBalancer.Host); err != nil { - conditions.MarkFalse(kkCluster, infrav1.ExternalLoadBalancerReadyCondition, infrav1.WaitForDNSNameResolveReason, clusterv1.ConditionSeverityInfo, "") - clusterScope.Info("Waiting on API server DNS name to resolve") - return reconcile.Result{RequeueAfter: 15 * time.Second}, nil //nolint:nilerr - } - conditions.MarkTrue(kkCluster, infrav1.ExternalLoadBalancerReadyCondition) - - kkCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{ - Host: clusterScope.ControlPlaneLoadBalancer().Host, - Port: clusterScope.APIServerPort(), - } - - kkCluster.Status.Ready = true - - if res, err := r.reconcileInPlaceUpgrade(ctx, clusterScope); !res.IsZero() || err != nil { - return res, err - } - - if res, err := r.reconcilePatchAnnotations(ctx, clusterScope); !res.IsZero() || err != nil { - return res, err - } - - return ctrl.Result{}, nil -} - -func (r *Reconciler) reconcileInPlaceUpgrade(ctx context.Context, clusterScope *scope.ClusterScope) (ctrl.Result, error) { - kkCluster := clusterScope.KKCluster - if _, ok := kkCluster.GetAnnotations()[infrav1.InPlaceUpgradeVersionAnnotation]; !ok { - return ctrl.Result{}, nil - } - - clusterScope.Info("Reconcile KKCluster in-place upgrade") - - if !kkCluster.Status.Ready { - return ctrl.Result{}, nil - } - - cluster := clusterScope.Cluster - phases := []func(context.Context, *scope.ClusterScope) (ctrl.Result, error){ - // pause the cluster - func(ctx context.Context, clusterScope *scope.ClusterScope) (ctrl.Result, error) { - return r.reconcilePausedCluster(ctx, clusterScope, true) - }, - // set up the KKInstance annotations - func(ctx context.Context, clusterScope *scope.ClusterScope) (ctrl.Result, error) { - return r.reconcileKKInstanceInPlaceUpgrade(ctx, clusterScope, collections.ActiveKKInstances, - collections.OwnedKKInstances(kkCluster), - collections.ControlPlaneKKInstances(cluster.Name)) - }, - func(ctx context.Context, clusterScope *scope.ClusterScope) (ctrl.Result, error) { - return r.reconcileKKInstanceInPlaceUpgrade(ctx, clusterScope, collections.ActiveKKInstances, - collections.OwnedKKInstances(kkCluster), - collections.Not(collections.ControlPlaneKKInstances(cluster.Name))) - }, - r.reconcileKKInstanceUpgradeCheck, - // patch the cluster-api resource .spec.version - r.reconcilePatchResourceSpecVersion, - // if upgrade is done, unpause the cluster - func(ctx context.Context, clusterScope *scope.ClusterScope) (ctrl.Result, error) { - return r.reconcilePausedCluster(ctx, clusterScope, false) - }, - } - - for _, phase := range phases { - // Call the inner reconciliation methods. - if phaseResult, err := phase(ctx, clusterScope); !phaseResult.IsZero() || err != nil { - return phaseResult, err - } - } - return ctrl.Result{}, nil -} - -func (r *Reconciler) requeueKKClusterForUnpausedCluster(ctx context.Context, log logr.Logger) handler.MapFunc { - return func(o client.Object) []ctrl.Request { - c, ok := o.(*clusterv1.Cluster) - if !ok { - panic(fmt.Sprintf("Expected a Cluster but got a %T", o)) - } - - log := log.WithValues("objectMapper", "clusterToKKCluster", "namespace", c.Namespace, "cluster", c.Name) - - // Don't handle deleted clusters - if !c.ObjectMeta.DeletionTimestamp.IsZero() { - log.V(4).Info("Cluster has a deletion timestamp, skipping mapping.") - return nil - } - - // Make sure the ref is set - if c.Spec.InfrastructureRef == nil { - log.V(4).Info("Cluster does not have an InfrastructureRef, skipping mapping.") - return nil - } - - if c.Spec.InfrastructureRef.GroupVersionKind().Kind != "KKCluster" { - log.V(4).Info("Cluster has an InfrastructureRef for a different type, skipping mapping.") - return nil - } - - kkCluster := &infrav1.KKCluster{} - key := types.NamespacedName{Namespace: c.Spec.InfrastructureRef.Namespace, Name: c.Spec.InfrastructureRef.Name} - - if err := r.Get(ctx, key, kkCluster); err != nil { - log.V(4).Error(err, "Failed to get KubeKey cluster") - return nil - } - - if annotations.IsExternallyManaged(kkCluster) { - log.V(4).Info("KKCluster is externally managed, skipping mapping.") - return nil - } - - log.V(4).Info("Adding request.", "kkCluster", c.Spec.InfrastructureRef.Name) - return []ctrl.Request{ - { - NamespacedName: client.ObjectKey{Namespace: c.Namespace, Name: c.Spec.InfrastructureRef.Name}, - }, - } - } -} diff --git a/controllers/kkcluster/kkcluster_controller_phases.go b/controllers/kkcluster/kkcluster_controller_phases.go deleted file mode 100644 index b1ffdb4b..00000000 --- a/controllers/kkcluster/kkcluster_controller_phases.go +++ /dev/null @@ -1,181 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kkcluster - -import ( - "context" - "fmt" - "time" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/util/errors" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/cluster-api/util/conditions" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" - "github.com/kubesphere/kubekey/v3/pkg/scope" - "github.com/kubesphere/kubekey/v3/util/collections" -) - -func (r *Reconciler) reconcilePausedCluster(ctx context.Context, clusterScope *scope.ClusterScope, paused bool) (ctrl.Result, error) { - action := "false" - if paused { - action = "true" - } - - clusterScope.Info(fmt.Sprintf("Reconcile KKCluster paused=%s cluster", action)) - - kkCluster := clusterScope.KKCluster - cluster, err := util.GetOwnerCluster(ctx, r.Client, kkCluster.ObjectMeta) - if err != nil { - return reconcile.Result{}, err - } - - if cluster.Spec.Paused == paused { - clusterScope.V(4).Info(fmt.Sprintf("Skipping reconcilePausedCluster because cluster is already paused=%s", action)) - return ctrl.Result{}, nil - } - - cluster.Spec.Paused = paused - if err := r.Client.Update(ctx, cluster); err != nil { - clusterScope.Error(err, fmt.Sprintf("failed to make the cluster paused=%s, will requeue", action)) - return ctrl.Result{RequeueAfter: 15 * time.Second}, err - } - return ctrl.Result{}, nil -} - -func (r *Reconciler) reconcileKKInstanceInPlaceUpgrade(ctx context.Context, clusterScope *scope.ClusterScope, - filters ...collections.Func) (ctrl.Result, error) { - clusterScope.Info("Reconcile KKCluster KKInstance in-place upgrade") - - kkCluster := clusterScope.KKCluster - annotation := kkCluster.GetAnnotations() - newVersion, ok := annotation[infrav1.InPlaceUpgradeVersionAnnotation] - if !ok { - clusterScope.V(4).Info("Skipping reconcileKKInstanceAnnotations because kkCluster does not have annotation %s", infrav1.InPlaceUpgradeVersionAnnotation) - return ctrl.Result{}, nil - } - - kkInstances, err := collections.GetFilteredKKInstancesForKKCluster(ctx, r.Client, kkCluster, filters...) - if err != nil { - clusterScope.Error(err, "failed to get active kkInstances for kkCluster") - return ctrl.Result{}, err - } - - if len(kkInstances) == 0 { - return ctrl.Result{}, errors.Errorf("no kkInstances found for upgrade") - } - - for _, kkInstance := range kkInstances { - if _, ok := kkInstance.Annotations[infrav1.InPlaceUpgradeVersionAnnotation]; !ok { - kkiCopy := kkInstance.DeepCopy() - kkiCopy.Annotations[infrav1.InPlaceUpgradeVersionAnnotation] = newVersion - - clusterScope.Info("Patch the KKInstance", "KKInstance", kkiCopy.Name, "annotation", kkiCopy.Annotations) - if err := r.Client.Update(ctx, kkiCopy); err != nil { - clusterScope.Error(err, "failed to patch annotation for kkInstance %s", kkiCopy.Name) - conditions.MarkFalse(kkCluster, infrav1.CallKKInstanceInPlaceUpgradeCondition, infrav1.KKInstanceObjectNotUpdatedReason, - clusterv1.ConditionSeverityWarning, "Failed to update annotation for kkInstance %s", kkiCopy.Name) - r.Recorder.Eventf(kkCluster, corev1.EventTypeWarning, "FailedUpdateKKInstance", - "Failed to update kkInstance %s annotation: %v", kkiCopy.Name, err) - return ctrl.Result{RequeueAfter: 15 * time.Second}, err - } - } - } - conditions.MarkTrue(kkCluster, infrav1.CallKKInstanceInPlaceUpgradeCondition) - return ctrl.Result{}, nil -} - -func (r *Reconciler) reconcileKKInstanceUpgradeCheck(ctx context.Context, clusterScope *scope.ClusterScope) (ctrl.Result, error) { - clusterScope.Info("Reconcile KKCluster KKInstance in-place upgrade check") - - kkCluster := clusterScope.KKCluster - annotation := kkCluster.GetAnnotations() - if _, ok := annotation[infrav1.InPlaceUpgradeVersionAnnotation]; !ok { - clusterScope.V(4).Info("Skipping reconcileKKInstanceAnnotations because kkCluster does not have annotation %s", infrav1.InPlaceUpgradeVersionAnnotation) - return ctrl.Result{}, nil - } - - kkInstances, err := collections.GetFilteredKKInstancesForKKCluster(ctx, r.Client, kkCluster, collections.ActiveKKInstances, - collections.OwnedKKInstances(kkCluster)) - if err != nil { - clusterScope.Error(err, "failed to get active kkInstances for kkCluster") - return ctrl.Result{}, err - } - - conditions.MarkFalse(kkCluster, infrav1.AllKKInstancesUpgradeCompletedCondition, infrav1.WaitingForKKInstancesUpgradeReason, - clusterv1.ConditionSeverityInfo, "Waiting for all kkInstances upgrade completed") - for _, kkInstance := range kkInstances { - if result, err := r.upgradeChecks(ctx, clusterScope, kkInstance); !result.IsZero() || err != nil { - return result, err - } - } - conditions.MarkTrue(kkCluster, infrav1.AllKKInstancesUpgradeCompletedCondition) - return ctrl.Result{}, nil -} - -func (r *Reconciler) upgradeChecks(_ context.Context, clusterScope *scope.ClusterScope, kkInstance *infrav1.KKInstance) (ctrl.Result, error) { - clusterScope.V(4).Info("Reconcile KKCluster upgrade checks") - - newVersion, ok := kkInstance.GetAnnotations()[infrav1.InPlaceUpgradeVersionAnnotation] - if !ok { - return ctrl.Result{RequeueAfter: upgradeCheckFailedRequeueAfter}, - errors.Errorf("kkInstance %s does not have annotation %s", kkInstance.Name, infrav1.InPlaceUpgradeVersionAnnotation) - } - - allKKInstanceUpgradeConditions := []clusterv1.ConditionType{ - infrav1.KKInstanceInPlaceUpgradedCondition, - } - - var kkInstanceErrors []error - for _, condition := range allKKInstanceUpgradeConditions { - if err := upgradeChecksCondition("kkinstance", kkInstance, condition); err != nil { - kkInstanceErrors = append(kkInstanceErrors, err) - } - if newVersion != kkInstance.Status.NodeInfo.KubeletVersion { - kkInstanceErrors = append(kkInstanceErrors, errors.Errorf("kkInstance %s is still upgrading", kkInstance.Name)) - } - } - if len(kkInstanceErrors) > 0 { - aggregatedError := kerrors.NewAggregate(kkInstanceErrors) - r.Recorder.Eventf(clusterScope.KKCluster, corev1.EventTypeWarning, "KKInstanceInPlaceUpgradeUnFinished", - "Waiting for all KKInstances to pass upgrade checks") - clusterScope.Info("Waiting for KKInstance to pass upgrade checks", "failures", aggregatedError.Error()) - - return ctrl.Result{RequeueAfter: upgradeCheckFailedRequeueAfter}, nil - } - - return ctrl.Result{}, nil -} - -func upgradeChecksCondition(kind string, obj conditions.Getter, condition clusterv1.ConditionType) error { - c := conditions.Get(obj, condition) - if c == nil { - return errors.Errorf("%s %s does not have %s condition", kind, obj.GetName(), condition) - } - if c.Status == corev1.ConditionFalse { - return errors.Errorf("%s %s reports %s condition is false (%s, %s)", kind, obj.GetName(), condition, c.Severity, c.Message) - } - if c.Status == corev1.ConditionUnknown { - return errors.Errorf("%s %s reports %s condition is unknown (%s)", kind, obj.GetName(), condition, c.Message) - } - return nil -} diff --git a/controllers/kkcluster/patch.go b/controllers/kkcluster/patch.go deleted file mode 100644 index 47381881..00000000 --- a/controllers/kkcluster/patch.go +++ /dev/null @@ -1,249 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kkcluster - -import ( - "context" - "time" - - "github.com/pkg/errors" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/utils/pointer" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" - "sigs.k8s.io/cluster-api/util" - capicollections "sigs.k8s.io/cluster-api/util/collections" - "sigs.k8s.io/cluster-api/util/conditions" - "sigs.k8s.io/cluster-api/util/patch" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" - "github.com/kubesphere/kubekey/v3/pkg/scope" - "github.com/kubesphere/kubekey/v3/util/collections" -) - -func (r *Reconciler) reconcilePatchAnnotations(ctx context.Context, clusterScope *scope.ClusterScope) (res ctrl.Result, err error) { - clusterScope.Info("Reconcile KKCluster patch KKCluster and KKInstance") - - defer func() { - if err != nil && res.IsZero() { - clusterScope.Info("Reconcile KKCluster patch annotations failed, requeue") - res = ctrl.Result{RequeueAfter: 15 * time.Second} - } - }() - - kkCluster := clusterScope.KKCluster - kkInstances, err := collections.GetFilteredKKInstancesForKKCluster(ctx, r.Client, kkCluster, collections.ActiveKKInstances, - collections.OwnedKKInstances(kkCluster)) - if err != nil { - clusterScope.Error(err, "failed to get active kkInstances for kkCluster") - return ctrl.Result{}, err - } - - for _, kkInstance := range kkInstances { - if _, ok := kkInstance.GetAnnotations()[infrav1.InPlaceUpgradeVersionAnnotation]; ok { - clusterScope.V(4).Info("KKCluster patch annotations for kkInstance", - "kkInstance", kkInstance.Name) - // delete in-place upgrade annotation - kkInstanceAnnotation := kkInstance.GetAnnotations() - delete(kkInstanceAnnotation, infrav1.InPlaceUpgradeVersionAnnotation) - kkInstance.SetAnnotations(kkInstanceAnnotation) - - if err := r.Client.Update(ctx, kkInstance); err != nil { - clusterScope.Error(err, "failed to update kkInstance") - return ctrl.Result{}, err - } - } - - if conditions.Has(kkInstance, infrav1.KKInstanceInPlaceUpgradeBinariesCondition) { - clusterScope.V(4).Info("KKCluster patch conditions for kkInstance", - "kkInstance", kkInstance.Name) - // delete in-place upgrade condition - conditions.Delete(kkInstance, infrav1.KKInstanceInPlaceUpgradeBinariesCondition) - - if err := r.Client.Status().Update(ctx, kkInstance); err != nil { - clusterScope.Error(err, "failed to update kkInstance status") - return ctrl.Result{}, err - } - } - } - - annotation := kkCluster.GetAnnotations() - if _, ok := annotation[infrav1.InPlaceUpgradeVersionAnnotation]; ok { - clusterScope.V(4).Info("KKCluster patch annotations") - delete(annotation, infrav1.InPlaceUpgradeVersionAnnotation) - kkCluster.SetAnnotations(annotation) - patchHelper, err := patch.NewHelper(kkCluster, r.Client) - if err != nil { - return ctrl.Result{}, err - } - - if err = patchHelper.Patch(ctx, kkCluster); err != nil { - clusterScope.Error(err, "failed to patch KKCluster") - return ctrl.Result{}, err - } - } - return ctrl.Result{}, nil -} - -func (r *Reconciler) reconcilePatchResourceSpecVersion(ctx context.Context, clusterScope *scope.ClusterScope) (res ctrl.Result, err error) { - clusterScope.Info("Reconcile KKCluster patch resource spec version") - - defer func() { - if err != nil && res.IsZero() { - res = ctrl.Result{RequeueAfter: 15 * time.Second} - } - }() - - kkCluster := clusterScope.KKCluster - annotation := kkCluster.GetAnnotations() - newVersion, ok := annotation[infrav1.InPlaceUpgradeVersionAnnotation] - if !ok { - clusterScope.V(4).Info("Skipping reconcilePatchResourceSpecVersion because kkCluster does not have annotation %s", infrav1.InPlaceUpgradeVersionAnnotation) - return ctrl.Result{}, nil - } - - machines, err := clusterScope.GetMachines(ctx, capicollections.MatchesKubernetesVersion(newVersion)) - if err != nil { - clusterScope.Error(err, "failed to get machines for cluster") - return ctrl.Result{}, err - } - - clusterScope.Info("Prepare to patch the .spec.version of the machines' owner", "numMachine", machines.Len()) - - if res, err := r.patchKCPSpecVersion(ctx, clusterScope, newVersion, machines); !res.IsZero() || err != nil { - return res, err - } - - if res, err := r.patchMachineSet(ctx, clusterScope, newVersion, machines); !res.IsZero() || err != nil { - return res, err - } - - return ctrl.Result{}, nil -} - -func (r *Reconciler) patchKCPSpecVersion(ctx context.Context, clusterScope *scope.ClusterScope, newVersion string, machines capicollections.Machines) (ctrl.Result, error) { - cluster := clusterScope.Cluster - kcpRef := cluster.Spec.ControlPlaneRef - - if kcpRef == nil { - return ctrl.Result{}, nil - } - - kcp := &controlplanev1.KubeadmControlPlane{} - if err := r.Client.Get(ctx, client.ObjectKey{Namespace: kcpRef.Namespace, Name: kcpRef.Name}, kcp); err != nil { - if apierrors.IsNotFound(errors.Cause(err)) { - clusterScope.Info("Could not find KubeadmControlPlane, requeuing", "KubeadmControlPlane", kcpRef.Name) - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil - } - return ctrl.Result{RequeueAfter: 30 * time.Second}, err - } - - // can not match any upgraded machine - if machines.Filter(capicollections.OwnedMachines(kcp)).Len() == 0 { - clusterScope.Info("Could not find any upgraded control plane machine", "KubeadmControlPlane", kcp.Name) - return ctrl.Result{}, nil - } - - clusterScope.Info("Patching the KCP with the upgraded K8s version", "KubeadmControlPlane", kcp.Name) - kcp.Spec.Version = newVersion - if err := r.Client.Update(ctx, kcp); err != nil { - clusterScope.Error(err, "failed to patch KubeadmControlPlane %s", kcp.Name) - return ctrl.Result{}, err - } - return ctrl.Result{}, nil -} - -func (r *Reconciler) patchMachineSet(ctx context.Context, clusterScope *scope.ClusterScope, newVersion string, machines capicollections.Machines) (ctrl.Result, error) { - cluster := clusterScope.Cluster - machineSets := &clusterv1.MachineSetList{} - - if err := r.Client.List(ctx, machineSets, - client.InNamespace(cluster.Namespace), - client.MatchingLabels{clusterv1.ClusterLabelName: cluster.Name}); err != nil { - return ctrl.Result{}, err - } - - msMap := make(map[string]*clusterv1.MachineSet) - for i := range machineSets.Items { - ms := machineSets.Items[i] - if ms.Spec.ClusterName != cluster.Name { - continue - } - - if machines.Filter(capicollections.OwnedMachines(&ms)).Len() == 0 { - continue - } - - clusterScope.Info("Patching the machineSet with the upgraded K8s version", "MachineSet", ms.Name) - ms.Spec.Template.Spec.Version = pointer.String(newVersion) - if err := r.Client.Update(ctx, &ms); err != nil { - clusterScope.Error(err, "failed to patch MachineSet %s", ms.Name) - return ctrl.Result{}, err - } - msMap[ms.Name] = &ms - } - - if res, err := r.patchMachineDeployment(ctx, clusterScope, newVersion, msMap); !res.IsZero() || err != nil { - if err != nil { - return res, err - } - return res, nil - } - - return ctrl.Result{}, nil -} - -func (r *Reconciler) patchMachineDeployment(ctx context.Context, clusterScope *scope.ClusterScope, newVersion string, machinesets map[string]*clusterv1.MachineSet) (ctrl.Result, error) { - cluster := clusterScope.Cluster - machineDeployments := &clusterv1.MachineDeploymentList{} - - if err := r.Client.List(ctx, machineDeployments, - client.InNamespace(cluster.Namespace), - client.MatchingLabels{clusterv1.ClusterLabelName: cluster.Name}); err != nil { - return ctrl.Result{}, err - } - - for i := range machineDeployments.Items { - md := machineDeployments.Items[i] - if md.Spec.ClusterName != cluster.Name { - continue - } - - if !hasOwnGivenMachineSet(machinesets, &md) { - continue - } - - clusterScope.Info("Patching the machineDeployment with the upgraded K8s version", "MachineDeployment", md.Name) - md.Spec.Template.Spec.Version = pointer.String(newVersion) - if err := r.Client.Update(ctx, &md); err != nil { - clusterScope.Error(err, "failed to patch MachineDeployment %s", md.Name) - return ctrl.Result{}, err - } - } - return ctrl.Result{}, nil -} - -func hasOwnGivenMachineSet(machinesets map[string]*clusterv1.MachineSet, owner *clusterv1.MachineDeployment) bool { - for _, ms := range machinesets { - if util.IsOwnedByObject(ms, owner) { - return true - } - } - return false -} diff --git a/controllers/kkinstance/doc.go b/controllers/kkinstance/doc.go deleted file mode 100644 index 9a558fa9..00000000 --- a/controllers/kkinstance/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package kkinstance implements kkinstance controllers. -package kkinstance diff --git a/controllers/kkinstance/kkinstance_controller.go b/controllers/kkinstance/kkinstance_controller.go deleted file mode 100644 index 607582a9..00000000 --- a/controllers/kkinstance/kkinstance_controller.go +++ /dev/null @@ -1,523 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kkinstance - -import ( - "context" - "fmt" - "reflect" - "time" - - "github.com/go-logr/logr" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/tools/record" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/remote" - cutil "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/cluster-api/util/annotations" - "sigs.k8s.io/cluster-api/util/conditions" - "sigs.k8s.io/cluster-api/util/predicates" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/scope" - "github.com/kubesphere/kubekey/v3/pkg/service" - "github.com/kubesphere/kubekey/v3/pkg/service/binary" - "github.com/kubesphere/kubekey/v3/pkg/service/bootstrap" - "github.com/kubesphere/kubekey/v3/pkg/service/containermanager" - "github.com/kubesphere/kubekey/v3/pkg/service/provisioning" - "github.com/kubesphere/kubekey/v3/pkg/service/repository" - "github.com/kubesphere/kubekey/v3/util" -) - -const ( - controllerName = "kkinstance-controller" - - defaultRequeueWait = 30 * time.Second - defaultKKInstanceInterval = 5 * time.Second - defaultKKInstanceTimeout = 10 * time.Minute -) - -// Locker is a lock that is used around. -type Locker interface { - Lock(ctx context.Context, cluster *clusterv1.Cluster, kkInstance *infrav1.KKInstance) bool - Unlock(ctx context.Context, cluster *clusterv1.Cluster) bool -} - -// Reconciler reconciles a KKInstance object -type Reconciler struct { - client.Client - Scheme *runtime.Scheme - Tracker *remote.ClusterCacheTracker - Recorder record.EventRecorder - Lock Locker - sshClientFactory func(scope *scope.InstanceScope) ssh.Interface - bootstrapFactory func(sshClient ssh.Interface, scope scope.LBScope, instanceScope *scope.InstanceScope) service.Bootstrap - repositoryFactory func(sshClient ssh.Interface, scope scope.KKInstanceScope, instanceScope *scope.InstanceScope) service.Repository - binaryFactory func(sshClient ssh.Interface, scope scope.KKInstanceScope, instanceScope *scope.InstanceScope, distribution string) service.BinaryService - containerManagerFactory func(sshClient ssh.Interface, scope scope.KKInstanceScope, instanceScope *scope.InstanceScope) service.ContainerManager - provisioningFactory func(sshClient ssh.Interface, format bootstrapv1.Format) service.Provisioning - WatchFilterValue string - DataDir string - - WaitKKInstanceInterval time.Duration - WaitKKInstanceTimeout time.Duration -} - -func (r *Reconciler) getSSHClient(scope *scope.InstanceScope) ssh.Interface { - if r.sshClientFactory != nil { - return r.sshClientFactory(scope) - } - if scope.KKInstance.Spec.Auth.Secret != "" { - secret := &corev1.Secret{} - ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) - defer cancel() - if err := r.Get(ctx, types.NamespacedName{Namespace: scope.Cluster.Namespace, Name: scope.KKInstance.Spec.Auth.Secret}, secret); err == nil { - if scope.KKInstance.Spec.Auth.PrivateKey == "" { // replace PrivateKey by secret - scope.KKInstance.Spec.Auth.PrivateKey = string(secret.Data["privateKey"]) - } - if scope.KKInstance.Spec.Auth.Password == "" { // replace password by secret - scope.KKInstance.Spec.Auth.Password = string(secret.Data["password"]) - } - } - } - return ssh.NewClient(scope.KKInstance.Spec.Address, scope.KKInstance.Spec.Auth, &scope.Logger) -} - -func (r *Reconciler) getBootstrapService(sshClient ssh.Interface, scope scope.LBScope, instanceScope *scope.InstanceScope) service.Bootstrap { - if r.bootstrapFactory != nil { - return r.bootstrapFactory(sshClient, scope, instanceScope) - } - return bootstrap.NewService(sshClient, scope, instanceScope) -} - -func (r *Reconciler) getRepositoryService(sshClient ssh.Interface, scope scope.KKInstanceScope, instanceScope *scope.InstanceScope) service.Repository { - if r.repositoryFactory != nil { - return r.repositoryFactory(sshClient, scope, instanceScope) - } - return repository.NewService(sshClient, scope, instanceScope) -} - -func (r *Reconciler) getBinaryService(sshClient ssh.Interface, scope scope.KKInstanceScope, instanceScope *scope.InstanceScope, distribution string) service.BinaryService { - if r.binaryFactory != nil { - return r.binaryFactory(sshClient, scope, instanceScope, distribution) - } - return binary.NewService(sshClient, scope, instanceScope, distribution) -} - -func (r *Reconciler) getContainerManager(sshClient ssh.Interface, scope scope.KKInstanceScope, instanceScope *scope.InstanceScope) service.ContainerManager { - if r.containerManagerFactory != nil { - return r.containerManagerFactory(sshClient, scope, instanceScope) - } - return containermanager.NewService(sshClient, scope, instanceScope) -} - -func (r *Reconciler) getProvisioningService(sshClient ssh.Interface, format bootstrapv1.Format) service.Provisioning { - if r.provisioningFactory != nil { - return r.provisioningFactory(sshClient, format) - } - return provisioning.NewService(sshClient, format) -} - -// SetupWithManager sets up the controller with the Manager. -func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { - log := ctrl.LoggerFrom(ctx) - - if r.Lock == nil { - r.Lock = NewMutex(mgr.GetClient()) - } - if r.WaitKKInstanceInterval.Nanoseconds() == 0 { - r.WaitKKInstanceInterval = defaultKKInstanceInterval - } - if r.WaitKKInstanceTimeout.Nanoseconds() == 0 { - r.WaitKKInstanceTimeout = defaultKKInstanceTimeout - } - - c, err := ctrl.NewControllerManagedBy(mgr). - WithOptions(options). - For(&infrav1.KKInstance{}). - Watches( - &source.Kind{Type: &infrav1.KKMachine{}}, - handler.EnqueueRequestsFromMapFunc(r.KKMachineToKKInstanceMapFunc(log)), - ). - Watches( - &source.Kind{Type: &infrav1.KKCluster{}}, - handler.EnqueueRequestsFromMapFunc(r.KKClusterToKKInstances(log)), - ). - WithEventFilter(predicates.ResourceHasFilterLabel(log, r.WatchFilterValue)). - WithEventFilter( - predicate.Funcs{ - // Avoid reconciling if the event triggering the reconciliation is related to incremental status updates - // for KKInstance resources only - UpdateFunc: func(e event.UpdateEvent) bool { - log.V(5).Info("KKInstance controller update predicate") - if _, ok := e.ObjectOld.(*infrav1.KKInstance); !ok { - log.V(5).Info(fmt.Sprintf("gvk is %s, not equale KKInstance", e.ObjectOld.GetObjectKind().GroupVersionKind().Kind)) - return true - } - - oldInstance := e.ObjectOld.(*infrav1.KKInstance).DeepCopy() - newInstance := e.ObjectNew.(*infrav1.KKInstance).DeepCopy() - - oldInstance.Status = infrav1.KKInstanceStatus{} - newInstance.Status = infrav1.KKInstanceStatus{} - - oldInstance.ObjectMeta.ResourceVersion = "" - newInstance.ObjectMeta.ResourceVersion = "" - - if reflect.DeepEqual(oldInstance, newInstance) { - log.V(4).Info("oldInstance and newInstance are equaled, skip") - return false - } - log.V(4).Info("oldInstance and newInstance are not equaled, allowing further processing") - return true - }, - }, - ). - Build(r) - if err != nil { - return err - } - - err = c.Watch( - &source.Kind{Type: &clusterv1.Cluster{}}, - handler.EnqueueRequestsFromMapFunc(r.requeueKKInstancesForUnpausedCluster(log)), - predicates.ClusterUnpausedAndInfrastructureReady(log), - ) - if err != nil { - return err - } - - return nil -} - -//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=kkinstances;kkinstances/status;kkinstances/finalizers,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status,verbs=get;list;watch;update;patch -//+kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines;machines/status,verbs=get;list;watch;update;patch -// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="",resources=secrets;events;configmaps,verbs=get;list;watch;create;patch - -func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, retErr error) { - log := ctrl.LoggerFrom(ctx) - // Fetch the KKInstance. - kkInstance := &infrav1.KKInstance{} - err := r.Get(ctx, req.NamespacedName, kkInstance) - if err != nil { - if apierrors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - - // Fetch the KKMachine. - kkMachine, err := util.GetOwnerKKMachine(ctx, r.Client, kkInstance.ObjectMeta) - if err != nil { - return ctrl.Result{}, err - } - if kkMachine == nil { - log.Info("KKMachine Controller has not yet set OwnerRef") - return ctrl.Result{}, nil - } - - log = log.WithValues("kkMachine", kkMachine.Name) - - // Fetch the Machine. - machine, err := cutil.GetOwnerMachine(ctx, r.Client, kkMachine.ObjectMeta) - if err != nil { - return ctrl.Result{}, err - } - if machine == nil { - log.Info("Machine Controller has not yet set OwnerRef") - return ctrl.Result{}, nil - } - - log = log.WithValues("machine", machine.Name) - - // Fetch the Cluster. - cluster, err := cutil.GetClusterFromMetadata(ctx, r.Client, kkInstance.ObjectMeta) - if err != nil { - log.Info("KKInstance is missing cluster label or cluster does not exist") - return ctrl.Result{}, nil - } - - log = log.WithValues("cluster", cluster.Name) - - infraCluster, err := util.GetInfraCluster(ctx, r.Client, log, cluster, "kkinstance", r.DataDir) - if err != nil { - return ctrl.Result{}, errors.Wrapf(err, "error getting infra provider cluster object") - } - if infraCluster == nil { - log.Info("KKCluster is not ready yet") - return ctrl.Result{}, nil - } - - instanceScope, err := scope.NewInstanceScope(scope.InstanceScopeParams{ - Client: r.Client, - Logger: &log, - Cluster: cluster, - Machine: machine, - InfraCluster: infraCluster, - KKMachine: kkMachine, - KKInstance: kkInstance, - }) - if err != nil { - log.Error(err, "failed to create instance scope") - return ctrl.Result{}, err - } - - if err := r.reconcilePing(ctx, instanceScope); err != nil { - return ctrl.Result{}, errors.Wrapf(err, "failed to ping remote instance [%s]", kkInstance.Spec.Address) - } - - // Always close the scope when exiting this function, so we can persist any KKInstance changes. - defer func() { - if err := instanceScope.Close(); err != nil && retErr == nil { - log.Error(err, "failed to patch object") - retErr = err - } - }() - - if !kkInstance.ObjectMeta.DeletionTimestamp.IsZero() { - return r.reconcileDelete(ctx, instanceScope, infraCluster) - } - - return r.reconcileNormal(ctx, instanceScope, infraCluster, infraCluster) -} - -func (r *Reconciler) reconcileDelete(ctx context.Context, instanceScope *scope.InstanceScope, lbScope scope.LBScope) (ctrl.Result, error) { - log := ctrl.LoggerFrom(ctx) - log.V(4).Info("Reconcile KKInstance delete") - - if annotations.IsPaused(instanceScope.Cluster, instanceScope.KKInstance) { - log.Info("KKInstance or linked Cluster is marked as paused. Won't reconcile") - return ctrl.Result{}, nil - } - - if conditions.Get(instanceScope.KKInstance, infrav1.KKInstanceDeletingBootstrapCondition) == nil { - conditions.MarkFalse(instanceScope.KKInstance, infrav1.KKInstanceDeletingBootstrapCondition, - infrav1.CleaningReason, clusterv1.ConditionSeverityInfo, "Cleaning the node before deletion") - } - - if err := instanceScope.PatchObject(); err != nil { - instanceScope.Error(err, "unable to patch object") - return ctrl.Result{}, err - } - - sshClient := r.getSSHClient(instanceScope) - if err := r.reconcileDeletingBootstrap(ctx, sshClient, instanceScope, lbScope); err != nil { - instanceScope.Error(err, "failed to reconcile deleting bootstrap") - return ctrl.Result{}, nil - } - conditions.MarkTrue(instanceScope.KKInstance, infrav1.KKInstanceDeletingBootstrapCondition) - instanceScope.SetState(infrav1.InstanceStateCleaned) - instanceScope.Info("Reconcile KKInstance delete successful") - controllerutil.RemoveFinalizer(instanceScope.KKInstance, infrav1.InstanceFinalizer) - return ctrl.Result{}, nil -} - -func (r *Reconciler) reconcileNormal(ctx context.Context, instanceScope *scope.InstanceScope, lbScope scope.LBScope, kkInstanceScope scope.KKInstanceScope) (ctrl.Result, error) { - instanceScope.Info("Reconcile KKInstance normal") - - // If the KKInstance is in an error state, return early. - if instanceScope.HasFailed() { - instanceScope.Info("Error state detected, skipping reconciliation") - return ctrl.Result{}, nil - } - - if instanceScope.KKInstance.Labels == nil { - instanceScope.KKInstance.Labels = make(map[string]string) - } - - instanceScope.KKInstance.Labels[infrav1.KKClusterLabelName] = instanceScope.InfraCluster.InfraClusterName() - - // If the KKMachine doesn't have our finalizer, add it. - if controllerutil.AddFinalizer(instanceScope.KKInstance, infrav1.InstanceFinalizer) { - // Register the finalizer after first read operation from KK to avoid orphaning KK resources on delete - if err := instanceScope.PatchObject(); err != nil { - instanceScope.Error(err, "unable to patch object") - return ctrl.Result{}, err - } - } - - sshClient := r.getSSHClient(instanceScope) - - phases := r.phaseFactory(kkInstanceScope) - for _, phase := range phases { - pollErr := wait.PollImmediate(r.WaitKKInstanceInterval, r.WaitKKInstanceTimeout, func() (done bool, err error) { - if err := phase(ctx, sshClient, instanceScope, kkInstanceScope, lbScope); err != nil { - return false, err - } - return true, nil - }) - if pollErr != nil { - instanceScope.Error(pollErr, "failed to reconcile phase") - return ctrl.Result{RequeueAfter: defaultRequeueWait}, pollErr - } - } - - instanceScope.SetState(infrav1.InstanceStateRunning) - instanceScope.Info("Reconcile KKInstance normal successful") - - if res, err := r.reconcileNode(ctx, instanceScope); !res.IsZero() || err != nil { - return res, err - } - - if _, ok := instanceScope.KKInstance.GetAnnotations()[infrav1.InPlaceUpgradeVersionAnnotation]; ok { - return r.reconcileInPlaceUpgrade(ctx, instanceScope, kkInstanceScope) - } - - return ctrl.Result{}, nil -} - -func (r *Reconciler) reconcileInPlaceUpgrade(ctx context.Context, instanceScope *scope.InstanceScope, kkInstanceScope scope.KKInstanceScope) (ctrl.Result, error) { - instanceScope.V(4).Info("Reconcile KKInstance in-place upgrade") - - // check node is ready - if instanceScope.KKInstance.Status.NodeRef == nil { - return ctrl.Result{RequeueAfter: defaultRequeueWait}, nil - } - - phases := []func(context.Context, *scope.InstanceScope) (ctrl.Result, error){ - func(ctx context.Context, instanceScope *scope.InstanceScope) (ctrl.Result, error) { - return r.reconcileInPlaceBinaryService(ctx, instanceScope, kkInstanceScope) - }, - r.reconcileInPlaceKubeadmUpgrade, - } - - for _, phase := range phases { - // Call the inner reconciliation methods. - if phaseResult, err := phase(ctx, instanceScope); !phaseResult.IsZero() || err != nil { - return phaseResult, err - } - } - return ctrl.Result{}, nil -} - -// KKClusterToKKInstances is a handler.ToRequestsFunc to be used to enqeue requests for reconciliation of KKInstance. -func (r *Reconciler) KKClusterToKKInstances(log logr.Logger) handler.MapFunc { - log.V(4).Info("KKClusterToKKInstances") - return func(o client.Object) []ctrl.Request { - c, ok := o.(*infrav1.KKCluster) - if !ok { - panic(fmt.Sprintf("Expected a KKCluster but got a %T", o)) - } - - log := log.WithValues("objectMapper", "kkClusterToKKInstance", "namespace", c.Namespace, "kkCluster", c.Name) - - // Don't handle deleted KKClusters - if !c.ObjectMeta.DeletionTimestamp.IsZero() { - log.V(4).Info("KKCluster has a deletion timestamp, skipping mapping.") - return nil - } - - cluster, err := cutil.GetOwnerCluster(context.TODO(), r.Client, c.ObjectMeta) - switch { - case apierrors.IsNotFound(err) || cluster == nil: - log.V(4).Info("Cluster for KKCluster not found, skipping mapping.") - return nil - case err != nil: - log.Error(err, "Failed to get owning cluster, skipping mapping.") - return nil - } - - return r.requestsForCluster(log, cluster.Namespace, cluster.Name) - } -} - -func (r *Reconciler) requeueKKInstancesForUnpausedCluster(log logr.Logger) handler.MapFunc { - log.V(4).Info("requeueKKInstancesForUnpausedCluster") - return func(o client.Object) []ctrl.Request { - c, ok := o.(*clusterv1.Cluster) - if !ok { - panic(fmt.Sprintf("Expected a Cluster but got a %T", o)) - } - - log := log.WithValues("objectMapper", "clusterToKKInstance", "namespace", c.Namespace, "cluster", c.Name) - - // Don't handle deleted clusters - if !c.ObjectMeta.DeletionTimestamp.IsZero() { - log.V(4).Info("Cluster has a deletion timestamp, skipping mapping.") - return nil - } - - return r.requestsForCluster(log, c.Namespace, c.Name) - } -} - -func (r *Reconciler) requestsForCluster(log logr.Logger, namespace, name string) []ctrl.Request { - labels := map[string]string{clusterv1.ClusterLabelName: name} - kkMachineList := &infrav1.KKMachineList{} - if err := r.Client.List(context.TODO(), kkMachineList, client.InNamespace(namespace), client.MatchingLabels(labels)); err != nil { - log.Error(err, "Failed to get owned Machines, skipping mapping.") - return nil - } - - result := make([]ctrl.Request, 0, len(kkMachineList.Items)) - for _, m := range kkMachineList.Items { - log.WithValues("kkmachine", m.Name) - - if m.Spec.InstanceID == nil { - log.V(4).Info("KKMachine does not have a providerID, will not add to reconciliation request.") - continue - } - - log.WithValues("kkInstance", m.Spec.InstanceID) - log.V(4).Info("Adding KKInstance to reconciliation request.") - result = append(result, ctrl.Request{NamespacedName: client.ObjectKey{Namespace: m.Namespace, Name: *m.Spec.InstanceID}}) - } - return result -} - -// KKMachineToKKInstanceMapFunc returns a handler.ToRequestsFunc that watches for -// KKMachine events and returns reconciliation requests for an KKInstance object. -func (r *Reconciler) KKMachineToKKInstanceMapFunc(log logr.Logger) handler.MapFunc { - log.V(4).Info("KKMachineToKKInstanceMapFunc") - return func(o client.Object) []reconcile.Request { - m, ok := o.(*infrav1.KKMachine) - if !ok { - return nil - } - - if m.Spec.InstanceID == nil { - return nil - } - - return []reconcile.Request{ - { - NamespacedName: client.ObjectKey{ - Namespace: m.Namespace, - Name: *m.Spec.InstanceID, - }, - }, - } - } -} diff --git a/controllers/kkinstance/kkinstance_controller_noderef.go b/controllers/kkinstance/kkinstance_controller_noderef.go deleted file mode 100644 index c0cd1263..00000000 --- a/controllers/kkinstance/kkinstance_controller_noderef.go +++ /dev/null @@ -1,191 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kkinstance - -import ( - "context" - "fmt" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - "k8s.io/klog/v2" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/api/v1beta1/index" - "sigs.k8s.io/cluster-api/controllers/noderefutil" - "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/cluster-api/util/conditions" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/kubesphere/kubekey/v3/pkg/scope" -) - -var ( - // ErrNodeNotFound signals that a corev1.Node could not be found for the given provider id. - ErrNodeNotFound = errors.New("cannot find node with matching ProviderID") -) - -func (r *Reconciler) reconcileNode(ctx context.Context, instanceScope *scope.InstanceScope) (ctrl.Result, error) { - instanceScope.Info("Reconcile node") - - kkMachine := instanceScope.KKMachine - if kkMachine.Spec.ProviderID == nil || *kkMachine.Spec.ProviderID == "" { - instanceScope.Info("Waiting for KKMachine ProviderID to be available") - return ctrl.Result{}, nil - } - - providerID, err := noderefutil.NewProviderID(*kkMachine.Spec.ProviderID) - if err != nil { - return ctrl.Result{}, err - } - - cluster := instanceScope.Cluster - remoteClient, err := r.Tracker.GetClient(ctx, util.ObjectKey(cluster)) - if err != nil { - return ctrl.Result{}, err - } - - kkInstance := instanceScope.KKInstance - // Even if Status.NodeRef exists, continue to do the following checks to make sure Node is healthy - node, err := r.getNode(ctx, remoteClient, providerID) - if err != nil { - if err == ErrNodeNotFound { - // While a NodeRef is set in the status, failing to get that node means the node is deleted. - // If Status.NodeRef is not set before, node still can be in the provisioning state. - if kkInstance.Status.NodeRef != nil { - conditions.MarkFalse(kkInstance, clusterv1.MachineNodeHealthyCondition, clusterv1.NodeNotFoundReason, clusterv1.ConditionSeverityError, "") - return ctrl.Result{}, errors.Wrapf(err, "no matching Node for Machine %q in namespace %q", kkInstance.Name, kkInstance.Namespace) - } - conditions.MarkFalse(kkInstance, clusterv1.MachineNodeHealthyCondition, clusterv1.NodeProvisioningReason, clusterv1.ConditionSeverityWarning, "") - // No need to requeue here. Nodes emit an event that triggers reconciliation. - return ctrl.Result{}, nil - } - instanceScope.Error(err, "Failed to retrieve Node by ProviderID") - r.Recorder.Event(kkInstance, corev1.EventTypeWarning, "Failed to retrieve Node by ProviderID", err.Error()) - return ctrl.Result{}, err - } - - // Set the Machine NodeRef. - if kkInstance.Status.NodeRef == nil { - kkInstance.Status.NodeRef = &corev1.ObjectReference{ - Kind: node.Kind, - APIVersion: node.APIVersion, - Name: node.Name, - UID: node.UID, - } - r.Recorder.Event(kkInstance, corev1.EventTypeNormal, "SuccessfulSetNodeRef", kkInstance.Status.NodeRef.Name) - } - - // Set the NodeSystemInfo. - kkInstance.Status.NodeInfo = &node.Status.NodeInfo - - // Do the remaining node health checks, then set the node health to true if all checks pass. - status, message := summarizeNodeConditions(node) - if status == corev1.ConditionFalse { - conditions.MarkFalse(kkInstance, clusterv1.MachineNodeHealthyCondition, clusterv1.NodeConditionsFailedReason, clusterv1.ConditionSeverityWarning, message) - return ctrl.Result{}, nil - } - if status == corev1.ConditionUnknown { - conditions.MarkUnknown(kkInstance, clusterv1.MachineNodeHealthyCondition, clusterv1.NodeConditionsFailedReason, message) - return ctrl.Result{}, nil - } - - conditions.MarkTrue(kkInstance, clusterv1.MachineNodeHealthyCondition) - return ctrl.Result{}, nil -} - -// summarizeNodeConditions summarizes a Node's conditions and returns the summary of condition statuses and concatenate failed condition messages: -// if there is at least 1 semantically-negative condition, summarized status = False; -// if there is at least 1 semantically-positive condition when there is 0 semantically negative condition, summarized status = True; -// if all conditions are unknown, summarized status = Unknown. -// (semantically true conditions: NodeMemoryPressure/NodeDiskPressure/NodePIDPressure == false or Ready == true.) -func summarizeNodeConditions(node *corev1.Node) (corev1.ConditionStatus, string) { - semanticallyFalseStatus := 0 - unknownStatus := 0 - - message := "" - for _, condition := range node.Status.Conditions { - switch condition.Type { - case corev1.NodeMemoryPressure, corev1.NodeDiskPressure, corev1.NodePIDPressure: - if condition.Status != corev1.ConditionFalse { - message += fmt.Sprintf("Node condition %s is %s", condition.Type, condition.Status) + ". " - if condition.Status == corev1.ConditionUnknown { - unknownStatus++ - continue - } - semanticallyFalseStatus++ - } - case corev1.NodeReady: - if condition.Status != corev1.ConditionTrue { - message += fmt.Sprintf("Node condition %s is %s", condition.Type, condition.Status) + ". " - if condition.Status == corev1.ConditionUnknown { - unknownStatus++ - continue - } - semanticallyFalseStatus++ - } - } - } - if semanticallyFalseStatus > 0 { - return corev1.ConditionFalse, message - } - if semanticallyFalseStatus+unknownStatus < 4 { - return corev1.ConditionTrue, message - } - return corev1.ConditionUnknown, message -} - -func (r *Reconciler) getNode(ctx context.Context, c client.Reader, providerID *noderefutil.ProviderID) (*corev1.Node, error) { - log := ctrl.LoggerFrom(ctx, "providerID", providerID) - nodeList := corev1.NodeList{} - if err := c.List(ctx, &nodeList, client.MatchingFields{index.NodeProviderIDField: providerID.IndexKey()}); err != nil { - return nil, err - } - if len(nodeList.Items) == 0 { - // If for whatever reason the index isn't registered or available, we fallback to loop over the whole list. - nl := corev1.NodeList{} - for { - if err := c.List(ctx, &nl, client.Continue(nl.Continue)); err != nil { - return nil, err - } - - for key, node := range nl.Items { - nodeProviderID, err := noderefutil.NewProviderID(node.Spec.ProviderID) - if err != nil { - log.Error(err, "Failed to parse ProviderID", "Node", klog.KRef("", nl.Items[key].GetName())) - continue - } - - if providerID.Equals(nodeProviderID) { - return &node, nil - } - } - - if nl.Continue == "" { - break - } - } - - return nil, ErrNodeNotFound - } - - if len(nodeList.Items) != 1 { - return nil, fmt.Errorf("unexpectedly found more than one Node matching the providerID %s", providerID.String()) - } - - return &nodeList.Items[0], nil -} diff --git a/controllers/kkinstance/kkinstance_controller_phase.go b/controllers/kkinstance/kkinstance_controller_phase.go deleted file mode 100644 index 98ab489d..00000000 --- a/controllers/kkinstance/kkinstance_controller_phase.go +++ /dev/null @@ -1,310 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kkinstance - -import ( - "context" - "encoding/base64" - "fmt" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/util/conditions" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/scope" - "github.com/kubesphere/kubekey/v3/pkg/service" -) - -func (r *Reconciler) phaseFactory(kkInstanceScope scope.KKInstanceScope) []func(context.Context, ssh.Interface, - *scope.InstanceScope, scope.KKInstanceScope, scope.LBScope) error { - var phases []func(context.Context, ssh.Interface, *scope.InstanceScope, scope.KKInstanceScope, scope.LBScope) error - switch kkInstanceScope.Distribution() { - case infrav1.KUBERNETES: - phases = append(phases, - r.reconcileBootstrap, - r.reconcileRepository, - r.reconcileBinaryService, - r.reconcileContainerManager, - r.reconcileProvisioning, - ) - case infrav1.K3S: - phases = append(phases, - r.reconcileBootstrap, - r.reconcileRepository, - r.reconcileBinaryService, - r.reconcileProvisioning, - ) - } - return phases -} - -func (r *Reconciler) reconcilePing(_ context.Context, instanceScope *scope.InstanceScope) error { - instanceScope.Info("Reconcile ping") - - sshClient := r.getSSHClient(instanceScope) - var err error - for i := 0; i < 3; i++ { - err = sshClient.Ping() - if err == nil { - break - } - } - return err -} - -func (r *Reconciler) reconcileDeletingBootstrap(_ context.Context, sshClient ssh.Interface, instanceScope *scope.InstanceScope, - lbScope scope.LBScope) (err error) { - instanceScope.Info("Reconcile deleting bootstrap") - - defer func() { - if err != nil { - conditions.MarkFalse( - instanceScope.KKInstance, - infrav1.KKInstanceDeletingBootstrapCondition, - infrav1.KKInstanceClearEnvironmentFailedReason, - clusterv1.ConditionSeverityWarning, - err.Error(), - ) - } else { - conditions.MarkTrue(instanceScope.KKInstance, infrav1.KKInstanceDeletingBootstrapCondition) - } - }() - - instanceScope.SetState(infrav1.InstanceStateCleaning) - - svc := r.getBootstrapService(sshClient, lbScope, instanceScope) - if err := svc.KubeadmReset(instanceScope.ContainerManager().CRISocket); err != nil { - return err - } - if err := svc.ResetNetwork(); err != nil { - return err - } - if err := svc.RemoveFiles(); err != nil { - return err - } - if err := svc.DaemonReload(); err != nil { - return err - } - if err := svc.UninstallK3s(); err != nil { - return err - } - return nil -} - -func (r *Reconciler) reconcileBootstrap(_ context.Context, sshClient ssh.Interface, instanceScope *scope.InstanceScope, - _ scope.KKInstanceScope, lbScope scope.LBScope) (err error) { - defer func() { - if err != nil { - conditions.MarkFalse( - instanceScope.KKInstance, - infrav1.KKInstanceBootstrappedCondition, - infrav1.KKInstanceInitOSFailedReason, - clusterv1.ConditionSeverityWarning, - err.Error(), - ) - } else { - conditions.MarkTrue(instanceScope.KKInstance, infrav1.KKInstanceBootstrappedCondition) - } - }() - if conditions.IsTrue(instanceScope.KKInstance, infrav1.KKInstanceBootstrappedCondition) { - instanceScope.Info("Instance has been bootstrapped") - return nil - } - - instanceScope.Info("Reconcile bootstrap") - - instanceScope.SetState(infrav1.InstanceStateBootstrapping) - - svc := r.getBootstrapService(sshClient, lbScope, instanceScope) - - if err := svc.AddUsers(); err != nil { - return err - } - if err := svc.SetHostname(); err != nil { - return err - } - if err := svc.CreateDirectory(); err != nil { - return err - } - if err := svc.ResetTmpDirectory(); err != nil { - return err - } - if err := svc.ExecInitScript(); err != nil { - return err - } - return nil -} - -func (r *Reconciler) reconcileRepository(_ context.Context, sshClient ssh.Interface, instanceScope *scope.InstanceScope, - scope scope.KKInstanceScope, _ scope.LBScope) (err error) { - defer func() { - if err != nil { - conditions.MarkFalse( - instanceScope.KKInstance, - infrav1.KKInstanceRepositoryReadyCondition, - infrav1.KKInstanceRepositoryFailedReason, - clusterv1.ConditionSeverityWarning, - err.Error(), - ) - } else { - conditions.MarkTrue(instanceScope.KKInstance, infrav1.KKInstanceRepositoryReadyCondition) - } - }() - if conditions.IsTrue(instanceScope.KKInstance, infrav1.KKInstanceRepositoryReadyCondition) { - instanceScope.Info("Instance has been repository ready") - return nil - } - - instanceScope.Info("Reconcile repository") - - svc := r.getRepositoryService(sshClient, scope, instanceScope) - if err = svc.Check(); err != nil { - return err - } - if err = svc.Get(r.WaitKKInstanceTimeout); err != nil { - return err - } - - err = svc.MountISO() - defer func(svc service.Repository) { - _ = svc.UmountISO() - }(svc) - if err != nil { - return err - } - - if err = svc.UpdateAndInstall(); err != nil { - return err - } - return nil -} - -func (r *Reconciler) reconcileBinaryService(_ context.Context, sshClient ssh.Interface, instanceScope *scope.InstanceScope, - kkInstanceScope scope.KKInstanceScope, _ scope.LBScope) (err error) { - defer func() { - if err != nil { - conditions.MarkFalse( - instanceScope.KKInstance, - infrav1.KKInstanceBinariesReadyCondition, - infrav1.KKInstanceGetBinaryFailedReason, - clusterv1.ConditionSeverityError, - err.Error(), - ) - } else { - conditions.MarkTrue(instanceScope.KKInstance, infrav1.KKInstanceBinariesReadyCondition) - } - }() - if conditions.IsTrue(instanceScope.KKInstance, infrav1.KKInstanceBinariesReadyCondition) { - instanceScope.Info("Instance's binaries is already ready") - return nil - } - - instanceScope.Info("Reconcile binary service") - - svc := r.getBinaryService(sshClient, kkInstanceScope, instanceScope, kkInstanceScope.Distribution()) - if err := svc.Download(r.WaitKKInstanceTimeout); err != nil { - return err - } - return nil -} - -func (r *Reconciler) reconcileContainerManager(_ context.Context, sshClient ssh.Interface, instanceScope *scope.InstanceScope, - scope scope.KKInstanceScope, _ scope.LBScope) (err error) { - defer func() { - if err != nil { - conditions.MarkFalse( - instanceScope.KKInstance, - infrav1.KKInstanceCRIReadyCondition, - infrav1.KKInstanceInstallCRIFailedReason, - clusterv1.ConditionSeverityError, - err.Error(), - ) - } else { - conditions.MarkTrue(instanceScope.KKInstance, infrav1.KKInstanceCRIReadyCondition) - } - }() - - if conditions.IsTrue(instanceScope.KKInstance, infrav1.KKInstanceCRIReadyCondition) { - instanceScope.Info("Instance's CRI is already ready") - return nil - } - - instanceScope.Info("Reconcile container manager") - - svc := r.getContainerManager(sshClient, scope, instanceScope) - if svc.IsExist() { - instanceScope.V(2).Info(fmt.Sprintf("container manager %s is exist, skip installation", svc.Type())) - return nil - } - - if err := svc.Get(r.WaitKKInstanceTimeout); err != nil { - return err - } - if err := svc.Install(); err != nil { - return err - } - return nil -} - -func (r *Reconciler) reconcileProvisioning(ctx context.Context, sshClient ssh.Interface, instanceScope *scope.InstanceScope, - _ scope.KKInstanceScope, _ scope.LBScope) (err error) { - defer func() { - if err != nil { - conditions.MarkFalse( - instanceScope.KKInstance, - infrav1.KKInstanceProvisionedCondition, - infrav1.KKInstanceRunCloudConfigFailedReason, - clusterv1.ConditionSeverityError, - err.Error(), - ) - } else { - conditions.MarkTrue(instanceScope.KKInstance, infrav1.KKInstanceProvisionedCondition) - } - }() - - if conditions.IsTrue(instanceScope.KKInstance, infrav1.KKInstanceProvisionedCondition) { - instanceScope.Info("Instance has been provisioned") - return nil - } - - instanceScope.Info("Reconcile provisioning") - - bootstrapData, format, err := instanceScope.GetRawBootstrapDataWithFormat(ctx) - if err != nil { - instanceScope.Error(err, "failed to get bootstrap data") - r.Recorder.Event(instanceScope.KKInstance, corev1.EventTypeWarning, "FailedGetBootstrapData", err.Error()) - return err - } - - svc := r.getProvisioningService(sshClient, format) - - commands, err := svc.RawBootstrapDataToProvisioningCommands(bootstrapData) - if err != nil { - instanceScope.Error(err, "provisioning code failed to parse", "bootstrap-data", base64.StdEncoding.EncodeToString(bootstrapData)) - return errors.Wrap(err, "failed to join a control plane node with kubeadm") - } - - for _, command := range commands { - if _, err := sshClient.SudoCmd(command.String()); err != nil { - return errors.Wrapf(err, "failed to run cloud config") - } - } - return nil -} diff --git a/controllers/kkinstance/kkinstance_controller_upgrade.go b/controllers/kkinstance/kkinstance_controller_upgrade.go deleted file mode 100644 index fe1325de..00000000 --- a/controllers/kkinstance/kkinstance_controller_upgrade.go +++ /dev/null @@ -1,340 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kkinstance - -import ( - "context" - "fmt" - "time" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - kubedrain "k8s.io/kubectl/pkg/drain" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/noderefutil" - "sigs.k8s.io/cluster-api/controllers/remote" - "sigs.k8s.io/cluster-api/util" - capicollections "sigs.k8s.io/cluster-api/util/collections" - "sigs.k8s.io/cluster-api/util/conditions" - ctrl "sigs.k8s.io/controller-runtime" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" - "github.com/kubesphere/kubekey/v3/pkg/scope" -) - -func (r *Reconciler) reconcileInPlaceBinaryService(_ context.Context, instanceScope *scope.InstanceScope, kkInstanceScope scope.KKInstanceScope) (ctrl.Result, error) { - instanceScope.Info("Reconcile in-place upgrade binary service") - instanceScope.SetState(infrav1.InstanceStateInPlaceUpgrading) - - kkInstance := instanceScope.KKInstance - if conditions.IsTrue(kkInstance, infrav1.KKInstanceInPlaceUpgradeBinariesCondition) || - instanceScope.InPlaceUpgradeVersion() == kkInstance.Status.NodeInfo.KubeletVersion { - instanceScope.Info("Skipping reconcileInPlaceBinaryServiceNode") - return ctrl.Result{}, nil - } - - clusterScope := instanceScope.InfraCluster - svc := r.getBinaryService(r.getSSHClient(instanceScope), kkInstanceScope, instanceScope, clusterScope.Distribution()) - if err := svc.UpgradeDownload(r.WaitKKInstanceTimeout); err != nil { - conditions.MarkFalse(instanceScope.KKInstance, infrav1.KKInstanceInPlaceUpgradeBinariesCondition, - infrav1.KKInstanceInPlaceGetBinaryFailedReason, clusterv1.ConditionSeverityError, err.Error()) - return ctrl.Result{}, err - } - conditions.MarkTrue(instanceScope.KKInstance, infrav1.KKInstanceInPlaceUpgradeBinariesCondition) - r.Recorder.Event(instanceScope.KKInstance, corev1.EventTypeNormal, "InPlaceUpgradeDownloadBinaries", "In-place upgrade download binaries") - return ctrl.Result{}, nil -} - -func (r *Reconciler) reconcileInPlaceKubeadmUpgrade(ctx context.Context, instanceScope *scope.InstanceScope) (ctrl.Result, error) { - if instanceScope.Machine.Status.NodeRef == nil { - return ctrl.Result{RequeueAfter: defaultRequeueWait}, errors.Errorf("cannot upgrade machine %s/%s: nodeRef not found", instanceScope.Machine.Namespace, instanceScope.Machine.Name) - } - - nodeName := instanceScope.Machine.Status.NodeRef.Name - log := instanceScope.Logger.WithValues("Node", klog.KRef("", nodeName)) - log.Info("Reconcile in-place kubeadm upgrade") - - kkInstance := instanceScope.KKInstance - if instanceScope.InPlaceUpgradeVersion() == kkInstance.Status.NodeInfo.KubeletVersion { - log.V(4).Info("Skipping reconcileInPlaceKubeadmUpgrade, because node kubelet version is already the desired version") - instanceScope.SetState(infrav1.InstanceStateRunning) - conditions.MarkTrue(instanceScope.KKInstance, infrav1.KKInstanceInPlaceUpgradedCondition) - return ctrl.Result{}, nil - } - - cluster := instanceScope.Cluster - if res, err := r.kubeadmUpgrade(ctx, instanceScope); !res.IsZero() || err != nil { - return res, err - } - - // reconcile drain control plane node - if r.isNodeDrainAllowed(instanceScope) { - log.Info("Draining node") - - m := instanceScope.Machine - if conditions.Get(kkInstance, clusterv1.DrainingSucceededCondition) == nil { - conditions.MarkFalse(kkInstance, clusterv1.DrainingSucceededCondition, clusterv1.DrainingReason, clusterv1.ConditionSeverityInfo, "Draining the node before in-place upgrade") - } - - if result, err := r.drainNode(ctx, instanceScope); !result.IsZero() || err != nil { - if err != nil { - conditions.MarkFalse(m, clusterv1.DrainingSucceededCondition, clusterv1.DrainingFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) - r.Recorder.Eventf(m, corev1.EventTypeWarning, "FailedDrainNode", "error draining Machine's node %q: %v", m.Status.NodeRef.Name, err) - } - return result, err - } - - conditions.MarkTrue(kkInstance, clusterv1.DrainingSucceededCondition) - r.Recorder.Eventf(kkInstance, corev1.EventTypeNormal, "SuccessfulDrainNode", "success draining KKInstance's node %q", m.Status.NodeRef.Name) - } - - // reconcile restart control plane kubelet - if err := r.restartKubelet(ctx, instanceScope); err != nil { - log.Error(err, "Failed to restart kubelet") - r.Recorder.Eventf(instanceScope.KKInstance, corev1.EventTypeWarning, "FailedInPlaceRestartingKubelet", - "Failed to restart the kubelet for cluster %s/%s node: %v", cluster.Namespace, cluster.Name, err) - } - - // reconcile uncordon control plane - if r.isNodeDrainAllowed(instanceScope) { - if err := r.cordonOrUncordonNode(ctx, instanceScope, false); err != nil { - log.Error(err, "Failed to uncordon node") - r.Recorder.Eventf(instanceScope.KKInstance, corev1.EventTypeWarning, "FailedUncordonNode", - "Failed to uncordon node for cluster %s/%s node: %v", cluster.Namespace, cluster.Name, err) - } - } - - return ctrl.Result{RequeueAfter: defaultRequeueWait}, nil -} - -func (r *Reconciler) kubeadmUpgrade(ctx context.Context, instanceScope *scope.InstanceScope) (ctrl.Result, error) { - nodeName := instanceScope.Machine.Status.NodeRef.Name - log := instanceScope.Logger.WithValues("Node", klog.KRef("", nodeName)) - - cluster := instanceScope.Cluster - ownedMachine, err := instanceScope.InfraCluster.GetMachines(ctx) - if err != nil { - log.Error(err, "failed to retrieve owned machines for cluster") - return ctrl.Result{}, err - } - - desiredCPMMachine := ownedMachine.Filter(capicollections.ControlPlaneMachines(cluster.Name)) - desiredCPMReplicas := len(desiredCPMMachine) - numMachines := len(desiredCPMMachine.Filter(capicollections.MatchesKubernetesVersion(instanceScope.InPlaceUpgradeVersion()))) - log.V(4).Info("Number of control plane", "desiredReplicas", desiredCPMReplicas, "numMachines", numMachines) - switch { - case instanceScope.IsControlPlane(): - switch { - case numMachines < desiredCPMReplicas && numMachines == 0: - log.Info("In-place upgrading the first control plane", "Desired", desiredCPMReplicas, "Existing", numMachines) - return r.inPlaceUpgradeControlPlane(ctx, instanceScope) - case numMachines < desiredCPMReplicas && numMachines > 0: - log.Info("In-place upgrading other control plane", "Desired", desiredCPMReplicas, "Existing", numMachines) - return r.inPlaceUpgradeScaleUp(ctx, instanceScope) - } - case !instanceScope.IsControlPlane(): - if numMachines != desiredCPMReplicas { - log.Info("Waiting for control plane to be upgraded", "Desired", desiredCPMReplicas, "Existing", numMachines) - return ctrl.Result{RequeueAfter: 15 * time.Second}, nil - } - - desiredWorkerMachine := ownedMachine.Filter(capicollections.Not(capicollections.ControlPlaneMachines(cluster.Name))) - desiredWorkerReplicas := len(desiredWorkerMachine) - numWorker := len(desiredWorkerMachine.Filter(capicollections.MatchesKubernetesVersion(instanceScope.InPlaceUpgradeVersion()))) - log.V(4).Info("Number of worker", "desiredReplicas", desiredWorkerReplicas, "numMachines", numWorker) - - if numWorker < desiredWorkerReplicas { - log.Info("In-place upgrading worker nodes", "Desired", desiredWorkerReplicas, "Existing", numWorker) - if res, err := r.inPlaceUpgradeScaleUpWorker(ctx, instanceScope); !res.IsZero() || err != nil { - if err != nil { - log.Error(err, "Failed to in-place upgrade worker") - } - return res, err - } - } - } - return ctrl.Result{}, nil -} - -func (r *Reconciler) isNodeDrainAllowed(instanceScope *scope.InstanceScope) bool { - if _, exist := instanceScope.Machine.GetAnnotations()[clusterv1.ExcludeNodeDrainingAnnotation]; exist { - return false - } - if r.nodeDrainTimeoutExceeded(instanceScope.Machine) { - return false - } - return true -} - -func (r *Reconciler) nodeDrainTimeoutExceeded(machine *clusterv1.Machine) bool { - // if the NodeDrainTimeout type is not set by user - if machine.Spec.NodeDrainTimeout == nil || machine.Spec.NodeDrainTimeout.Seconds() <= 0 { - return false - } - - // if the draining succeeded condition does not exist - if conditions.Get(machine, clusterv1.DrainingSucceededCondition) == nil { - return false - } - - now := time.Now() - firstTimeDrain := conditions.GetLastTransitionTime(machine, clusterv1.DrainingSucceededCondition) - diff := now.Sub(firstTimeDrain.Time) - return diff.Seconds() >= machine.Spec.NodeDrainTimeout.Seconds() -} - -func (r *Reconciler) drainNode(ctx context.Context, instanceScope *scope.InstanceScope) (ctrl.Result, error) { - nodeName := instanceScope.Machine.Status.NodeRef.Name - log := instanceScope.Logger.WithValues("Node", klog.KRef("", nodeName)) - - cluster := instanceScope.Cluster - restConfig, err := remote.RESTConfig(ctx, controllerName, r.Client, util.ObjectKey(cluster)) - if err != nil { - log.Error(err, "Error creating a remote client while deleting Machine, won't retry") - return ctrl.Result{}, nil - } - kubeClient, err := kubernetes.NewForConfig(restConfig) - if err != nil { - log.Error(err, "Error creating a remote client while deleting Machine, won't retry") - return ctrl.Result{}, nil - } - - node, err := kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - // If an admin deletes the node directly, we'll end up here. - log.Error(err, "Could not find node from noderef, it may have already been upgrading") - return ctrl.Result{}, nil - } - return ctrl.Result{}, errors.Wrapf(err, "unable to get node %v", nodeName) - } - - drainer := &kubedrain.Helper{ - Client: kubeClient, - Ctx: ctx, - Force: true, - IgnoreAllDaemonSets: true, - DeleteEmptyDirData: true, - GracePeriodSeconds: -1, - // If a pod is not evicted in 20 seconds, retry the eviction next time the - // machine gets reconciled again (to allow other machines to be reconciled). - Timeout: 20 * time.Second, - OnPodDeletedOrEvicted: func(pod *corev1.Pod, usingEviction bool) { - verbStr := "Deleted" - if usingEviction { - verbStr = "Evicted" - } - log.Info(fmt.Sprintf("%s pod from Node", verbStr), - "Pod", klog.KObj(pod)) - }, - Out: writer{log.Info}, - ErrOut: writer{func(msg string, keysAndValues ...interface{}) { - log.Error(nil, msg, keysAndValues...) - }}, - } - - if noderefutil.IsNodeUnreachable(node) { - // When the node is unreachable and some pods are not evicted for as long as this timeout, we ignore them. - drainer.SkipWaitForDeleteTimeoutSeconds = 60 * 5 // 5 minutes - } - - if err := kubedrain.RunCordonOrUncordon(drainer, node, true); err != nil { - // Machine will be re-reconciled after a cordon failure. - log.Error(err, "Cordon failed") - return ctrl.Result{}, errors.Wrapf(err, "unable to cordon node %v", node.Name) - } - - if err := kubedrain.RunNodeDrain(drainer, node.Name); err != nil { - // Machine will be re-reconciled after a drain failure. - log.Error(err, "Drain failed, retry in 20s") - return ctrl.Result{RequeueAfter: 20 * time.Second}, nil - } - - log.Info("Drain successful") - return ctrl.Result{}, nil -} - -func (r *Reconciler) restartKubelet(_ context.Context, instanceScope *scope.InstanceScope) error { - nodeName := instanceScope.Machine.Status.NodeRef.Name - log := instanceScope.Logger.WithValues("Node", klog.KRef("", nodeName)) - log.Info("In-place restart kubelet") - - if _, err := r.getSSHClient(instanceScope).SudoCmdf("systemctl daemon-reload && systemctl restart kubelet"); err != nil { - return err - } - return nil -} - -func (r *Reconciler) cordonOrUncordonNode(ctx context.Context, instanceScope *scope.InstanceScope, desired bool) error { - nodeName := instanceScope.Machine.Status.NodeRef.Name - action := "cordon" - if !desired { - action = "uncordon" - } - - log := instanceScope.Logger.WithValues("Node", klog.KRef("", nodeName)) - log.Info(fmt.Sprintf("In-place %s node", action)) - - cluster := instanceScope.Cluster - restConfig, err := remote.RESTConfig(ctx, controllerName, r.Client, util.ObjectKey(cluster)) - if err != nil { - log.Error(err, fmt.Sprintf("Error creating a remote client while %s node, won't retry", action)) - return nil - } - kubeClient, err := kubernetes.NewForConfig(restConfig) - if err != nil { - log.Error(err, fmt.Sprintf("Error creating a remote client while %s bide, won't retry", action)) - return nil - } - - node, err := kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - // If an admin deletes the node directly, we'll end up here. - log.Error(err, "Could not find node from noderef, it may have already been upgrading") - return nil - } - return errors.Wrapf(err, "unable to get node %v", nodeName) - } - - drainer := &kubedrain.Helper{ - Client: kubeClient, - Ctx: ctx, - } - - if err := kubedrain.RunCordonOrUncordon(drainer, node, desired); err != nil { - // Machine will be re-reconciled after a cordon failure. - log.Error(err, fmt.Sprintf("%s failed", action)) - return errors.Wrapf(err, "unable to %s node %v", action, node.Name) - } - return nil -} - -// writer implements io.Writer interface as a pass-through for klog. -type writer struct { - logFunc func(msg string, keysAndValues ...interface{}) -} - -// Write passes string(p) into writer's logFunc and always returns len(p). -func (w writer) Write(p []byte) (n int, err error) { - w.logFunc(string(p)) - return len(p), nil -} diff --git a/controllers/kkinstance/kkinstance_init_mutex.go b/controllers/kkinstance/kkinstance_init_mutex.go deleted file mode 100644 index 5439197f..00000000 --- a/controllers/kkinstance/kkinstance_init_mutex.go +++ /dev/null @@ -1,191 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kkinstance - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog/v2" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" -) - -const semaphoreInformationKey = "lock-information" - -// Mutex uses a ConfigMap to synchronize KKInstance. -type Mutex struct { - client client.Client -} - -// NewMutex returns a lock that can be held by a KKInstance. -func NewMutex(client client.Client) *Mutex { - return &Mutex{ - client: client, - } -} - -// Lock allows a control plane node to be the first and only node to run kubeadm init. -func (m *Mutex) Lock(ctx context.Context, cluster *clusterv1.Cluster, kkInstance *infrav1.KKInstance) bool { - sema := newSemaphore() - cmName := configMapName(cluster.Name) - log := ctrl.LoggerFrom(ctx, "ConfigMap", klog.KRef(cluster.Namespace, cmName)) - err := m.client.Get(ctx, client.ObjectKey{ - Namespace: cluster.Namespace, - Name: cmName, - }, sema.ConfigMap) - switch { - case apierrors.IsNotFound(err): - break - case err != nil: - log.Error(err, "Failed to acquire init lock") - return false - default: // Successfully found an existing config map. - info, err := sema.information() - if err != nil { - log.Error(err, "Failed to get information about the existing init lock") - return false - } - // The kkinstance requesting the lock is the kkinstance that created the lock, therefore the lock is acquired. - if info.KKInstanceName == kkInstance.Name { - return true - } - - // If the kkinstance that created the lock can not be found unlock the mutex. - if err := m.client.Get(ctx, client.ObjectKey{ - Namespace: cluster.Namespace, - Name: info.KKInstanceName, - }, &clusterv1.Machine{}); err != nil { - log.Error(err, "Failed to get kkinstance holding lock") - if apierrors.IsNotFound(err) { - m.Unlock(ctx, cluster) - } - } - log.Info(fmt.Sprintf("Waiting for KKInstance %s to reconcile", info.KKInstanceName)) - return false - } - - // Adds owner reference, namespace and name - sema.setMetadata(cluster) - // Adds the additional information - if err := sema.setInformation(&information{KKInstanceName: kkInstance.Name}); err != nil { - log.Error(err, "Failed to acquire init lock while setting semaphore information") - return false - } - - log.Info("Attempting to acquire the lock") - err = m.client.Create(ctx, sema.ConfigMap) - switch { - case apierrors.IsAlreadyExists(err): - log.Info("Cannot acquire the lock. The lock has been acquired by someone else") - return false - case err != nil: - log.Error(err, "Error acquiring the init lock") - return false - default: - return true - } -} - -// Unlock releases the lock. -func (m *Mutex) Unlock(ctx context.Context, cluster *clusterv1.Cluster) bool { - sema := newSemaphore() - cmName := configMapName(cluster.Name) - log := ctrl.LoggerFrom(ctx, "ConfigMap", klog.KRef(cluster.Namespace, cmName)) - err := m.client.Get(ctx, client.ObjectKey{ - Namespace: cluster.Namespace, - Name: cmName, - }, sema.ConfigMap) - switch { - case apierrors.IsNotFound(err): - log.Info("Control plane lock not found, it may have been released already") - return true - case err != nil: - log.Error(err, "Error unlocking the control plane lock") - return false - default: - // Delete the config map semaphore if there is no error fetching it - if err := m.client.Delete(ctx, sema.ConfigMap); err != nil { - if apierrors.IsNotFound(err) { - return true - } - log.Error(err, "Error deleting the config map underlying the control plane lock") - return false - } - return true - } -} - -type information struct { - KKInstanceName string `json:"kkInstanceName"` -} - -type semaphore struct { - *corev1.ConfigMap -} - -func newSemaphore() *semaphore { - return &semaphore{&corev1.ConfigMap{}} -} - -func configMapName(clusterName string) string { - return fmt.Sprintf("%s-lock", clusterName) -} - -func (s semaphore) information() (*information, error) { - li := &information{} - if err := json.Unmarshal([]byte(s.Data[semaphoreInformationKey]), li); err != nil { - return nil, errors.Wrap(err, "failed to unmarshal semaphore information") - } - return li, nil -} - -func (s semaphore) setInformation(information *information) error { - b, err := json.Marshal(information) - if err != nil { - return errors.Wrap(err, "failed to marshal semaphore information") - } - s.Data = map[string]string{} - s.Data[semaphoreInformationKey] = string(b) - return nil -} - -func (s *semaphore) setMetadata(cluster *clusterv1.Cluster) { - s.ObjectMeta = metav1.ObjectMeta{ - Namespace: cluster.Namespace, - Name: configMapName(cluster.Name), - Labels: map[string]string{ - clusterv1.ClusterLabelName: cluster.Name, - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: cluster.APIVersion, - Kind: cluster.Kind, - Name: cluster.Name, - UID: cluster.UID, - }, - }, - } -} diff --git a/controllers/kkinstance/scale.go b/controllers/kkinstance/scale.go deleted file mode 100644 index 971ca8ca..00000000 --- a/controllers/kkinstance/scale.go +++ /dev/null @@ -1,154 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kkinstance - -import ( - "context" - "time" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/klog/v2" - "k8s.io/utils/pointer" - capicollections "sigs.k8s.io/cluster-api/util/collections" - ctrl "sigs.k8s.io/controller-runtime" - - "github.com/kubesphere/kubekey/v3/pkg/scope" -) - -func (r *Reconciler) inPlaceUpgradeControlPlane(ctx context.Context, instanceScope *scope.InstanceScope) (_ ctrl.Result, retErr error) { - nodeName := instanceScope.Machine.Status.NodeRef.Name - log := instanceScope.Logger.WithValues("Node", klog.KRef("", nodeName)) - - if !instanceScope.IsControlPlane() { - return ctrl.Result{}, nil - } - - cluster := instanceScope.Cluster - newVersionCPM, err := instanceScope.InfraCluster.GetMachines(ctx, capicollections.ControlPlaneMachines(cluster.Name), - capicollections.MatchesKubernetesVersion(instanceScope.InPlaceUpgradeVersion())) - if err != nil { - log.Error(err, "Failed to perform an uncached read of new version control plane machines") - return ctrl.Result{}, err - } - - if len(newVersionCPM) > 0 { - return ctrl.Result{}, errors.Errorf( - "already have the first control plane upgrade completed, found %d new version control plane for cluster %s/%s: controller cache or management cluster is misbehaving", - len(newVersionCPM), cluster.Namespace, cluster.Name, - ) - } - - // acquire the lock so that only the first machine configured - // as control plane get processed here - // if not the first, requeue - if !r.Lock.Lock(ctx, cluster, instanceScope.KKInstance) { - log.Info("A control plane is already upgrading, requeueing until the first control plane is upgraded") - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil - } - - defer func() { - if retErr != nil { - if !r.Lock.Unlock(ctx, cluster) { - retErr = kerrors.NewAggregate([]error{retErr, errors.New("failed to unlock the lock")}) - } - } - }() - - if _, err := r.getSSHClient(instanceScope).SudoCmdf("kubeadm upgrade apply %s -y "+ - "--ignore-preflight-errors=all "+ - "--allow-experimental-upgrades "+ - "--allow-release-candidate-upgrades "+ - "--etcd-upgrade=false "+ - "--certificate-renewal=true", instanceScope.InPlaceUpgradeVersion()); err != nil { - log.Error(err, "Failed to upgrade the first control plane") - r.Recorder.Eventf(instanceScope.KKInstance, corev1.EventTypeWarning, "FailedInPlaceUpgradeControlPlane", - "Failed to upgrade the first control plane for cluster %s/%s control plane: %v", cluster.Namespace, cluster.Name, err) - return ctrl.Result{}, err - } - - // reconcile patch machine spec version - log.V(4).Info("Patching machine spec version") - if err := r.patchMachineSpecVersion(ctx, instanceScope); err != nil { - log.Error(err, "Failed to patch machine spec version") - r.Recorder.Eventf(instanceScope.KKInstance, corev1.EventTypeWarning, "FailedInPlaceUpgradeControlPlane", - "Failed to patch machine %s spec version for cluster %s/%s machine: %v", instanceScope.Machine, cluster.Namespace, cluster.Name, err) - return ctrl.Result{}, err - } - - return ctrl.Result{Requeue: true}, nil -} - -func (r *Reconciler) inPlaceUpgradeScaleUp(ctx context.Context, instanceScope *scope.InstanceScope) (ctrl.Result, error) { - nodeName := instanceScope.Machine.Status.NodeRef.Name - log := instanceScope.Logger.WithValues("Node", klog.KRef("", nodeName)) - - if instanceScope.InPlaceUpgradeVersion() == *instanceScope.Machine.Spec.Version { - return ctrl.Result{RequeueAfter: 15 * time.Second}, nil - } - - cluster := instanceScope.Cluster - r.Lock.Unlock(ctx, cluster) - - if _, err := r.getSSHClient(instanceScope).SudoCmdf("kubeadm upgrade node"); err != nil { - log.Error(err, "Failed to upgrade the control plane") - r.Recorder.Eventf(instanceScope.KKInstance, corev1.EventTypeWarning, "FailedInPlaceUpgradeControlPlane", - "Failed to upgrade the node for cluster %s/%s control plane: %v", cluster.Namespace, cluster.Name, err) - return ctrl.Result{}, err - } - - // reconcile patch machine spec version - log.V(4).Info("Patching machine spec version") - if err := r.patchMachineSpecVersion(ctx, instanceScope); err != nil { - log.Error(err, "Failed to patch machine spec version") - r.Recorder.Eventf(instanceScope.KKInstance, corev1.EventTypeWarning, "FailedInPlaceUpgradeControlPlane", - "Failed to patch machine %s spec version for cluster %s/%s machine: %v", instanceScope.Machine, cluster.Namespace, cluster.Name, err) - return ctrl.Result{}, err - } - - return ctrl.Result{Requeue: true}, nil -} - -func (r *Reconciler) inPlaceUpgradeScaleUpWorker(ctx context.Context, instanceScope *scope.InstanceScope) (ctrl.Result, error) { - nodeName := instanceScope.Machine.Status.NodeRef.Name - log := instanceScope.Logger.WithValues("Node", klog.KRef("", nodeName)) - - cluster := instanceScope.Cluster - if _, err := r.getSSHClient(instanceScope).SudoCmdf("kubeadm upgrade node"); err != nil { - log.Error(err, "Failed to upgrade the node") - r.Recorder.Eventf(instanceScope.KKInstance, corev1.EventTypeWarning, "FailedInPlaceUpgradeControlPlane", - "Failed to upgrade the node for cluster %s/%s worker: %v", cluster.Namespace, cluster.Name, err) - return ctrl.Result{}, err - } - - // reconcile patch machine spec version - log.V(4).Info("Patching machine spec version") - if err := r.patchMachineSpecVersion(ctx, instanceScope); err != nil { - log.Error(err, "Failed to patch machine spec version") - r.Recorder.Eventf(instanceScope.KKInstance, corev1.EventTypeWarning, "FailedInPlaceUpgradeControlPlane", - "Failed to patch machine %s spec version for cluster %s/%s machine: %v", instanceScope.Machine, cluster.Namespace, cluster.Name, err) - return ctrl.Result{}, err - } - return ctrl.Result{}, nil -} - -func (r *Reconciler) patchMachineSpecVersion(ctx context.Context, instanceScope *scope.InstanceScope) error { - machineCopy := instanceScope.Machine.DeepCopy() - machineCopy.Spec.Version = pointer.String(instanceScope.InPlaceUpgradeVersion()) - return r.Client.Update(ctx, machineCopy) -} diff --git a/controllers/kkmachine/doc.go b/controllers/kkmachine/doc.go deleted file mode 100644 index f1ad7bbd..00000000 --- a/controllers/kkmachine/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package kkmachine implements kkmachine controllers. -package kkmachine diff --git a/controllers/kkmachine/helpers.go b/controllers/kkmachine/helpers.go deleted file mode 100644 index c354c005..00000000 --- a/controllers/kkmachine/helpers.go +++ /dev/null @@ -1,159 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kkmachine - -import ( - "context" - - "github.com/imdario/mergo" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - apimachinerytypes "k8s.io/apimachinery/pkg/types" - capierrors "sigs.k8s.io/cluster-api/errors" - capiutil "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/cluster-api/util/patch" - "sigs.k8s.io/controller-runtime/pkg/client" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" - "github.com/kubesphere/kubekey/v3/pkg/scope" -) - -func (r *Reconciler) createInstance(ctx context.Context, machineScope *scope.MachineScope, - kkInstanceScope scope.KKInstanceScope) (*infrav1.KKInstance, error) { - machineScope.Info("Creating KKInstance") - - if machineScope.Machine.Spec.Version == nil { - err := errors.New("Machine's spec.version must be defined") - machineScope.SetFailureReason(capierrors.CreateMachineError) - machineScope.SetFailureMessage(err) - return nil, err - } - - instanceSpec, err := r.getUnassignedInstanceSpec(machineScope, kkInstanceScope) - if err != nil { - return nil, err - } - - instanceID := instanceSpec.Name - gv := infrav1.GroupVersion - labels := machineScope.Machine.Labels - labels[infrav1.KKClusterLabelName] = kkInstanceScope.InfraClusterName() - instance := &infrav1.KKInstance{ - ObjectMeta: metav1.ObjectMeta{ - Name: instanceID, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(machineScope.KKMachine, kkMachineKind)}, - Namespace: machineScope.KKMachine.Namespace, - // todo: if need to use the kkmachine labels? - Labels: labels, - Annotations: machineScope.KKMachine.Annotations, - }, - TypeMeta: metav1.TypeMeta{ - Kind: gv.WithKind("KKInstance").Kind, - APIVersion: gv.String(), - }, - Spec: *instanceSpec, - } - - instance.OwnerReferences = capiutil.EnsureOwnerRef(instance.OwnerReferences, metav1.OwnerReference{ - APIVersion: infrav1.GroupVersion.String(), - Kind: "KKCluster", - Name: machineScope.InfraCluster.InfraClusterName(), - UID: machineScope.InfraCluster.KKCluster.UID, - }) - - if err := r.Client.Create(ctx, instance); err != nil { - return nil, err - } - - return instance, nil -} - -func (r *Reconciler) getUnassignedInstanceSpec(machineScope *scope.MachineScope, kkInstanceScope scope.KKInstanceScope) (*infrav1.KKInstanceSpec, error) { - var instanceSpecs []infrav1.KKInstanceSpec - if len(machineScope.GetRoles()) != 0 { - for _, role := range machineScope.GetRoles() { - instanceSpecs = append(instanceSpecs, kkInstanceScope.GetInstancesSpecByRole(role)...) - } - } - - // get all existing instances - instances, err := kkInstanceScope.AllInstances() - if err != nil { - return nil, errors.Wrapf(err, "failed to get all existing instance") - } - instancesMap := make(map[string]struct{}, 0) - for _, v := range instances { - instancesMap[v.Spec.InternalAddress] = struct{}{} - } - - for _, spec := range instanceSpecs { - if _, ok := instancesMap[spec.InternalAddress]; ok { - continue - } - - auth := kkInstanceScope.GlobalAuth().DeepCopy() - if err := mergo.Merge(&spec.Auth, auth); err != nil { - return nil, err - } - - spec.ContainerManager = *machineScope.KKMachine.Spec.ContainerManager.DeepCopy() - spec.Repository = machineScope.KKMachine.Spec.Repository.DeepCopy() - return &spec, nil - } - return nil, errors.New("unassigned instance not found") -} - -func (r *Reconciler) deleteInstance(ctx context.Context, instance *infrav1.KKInstance) error { - if err := r.Client.Delete(ctx, instance); err != nil { - if !apierrors.IsNotFound(err) { - return err - } - } - return nil -} - -func (r *Reconciler) setNodeProviderID(ctx context.Context, machineScope *scope.MachineScope, instance *infrav1.KKInstance) error { - // Usually a cloud provider will do this, but there is no kubekey-cloud provider. - // Requeue if there is an error, as this is likely momentary load balancer - // state changes during control plane provisioning. - remoteClient, err := r.Tracker.GetClient(ctx, client.ObjectKeyFromObject(machineScope.Cluster)) - if err != nil { - return errors.Wrap(err, "failed to generate workload cluster client") - } - - node := &corev1.Node{} - if err = remoteClient.Get(ctx, apimachinerytypes.NamespacedName{Name: instance.Name}, node); err != nil { - return errors.Wrap(err, "failed to retrieve node") - } - - machineScope.Info("Setting Kubernetes node providerID") - - patchHelper, err := patch.NewHelper(node, remoteClient) - if err != nil { - return err - } - - node.Spec.ProviderID = machineScope.GetProviderID() - - if err = patchHelper.Patch(ctx, node); err != nil { - return errors.Wrap(err, "failed update providerID") - } - - return nil -} diff --git a/controllers/kkmachine/kkmachine_controller.go b/controllers/kkmachine/kkmachine_controller.go deleted file mode 100644 index a8046a11..00000000 --- a/controllers/kkmachine/kkmachine_controller.go +++ /dev/null @@ -1,491 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kkmachine - -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/go-logr/logr" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/tools/record" - "k8s.io/utils/pointer" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/noderefutil" - "sigs.k8s.io/cluster-api/controllers/remote" - capierrors "sigs.k8s.io/cluster-api/errors" - cutil "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/cluster-api/util/annotations" - "sigs.k8s.io/cluster-api/util/conditions" - "sigs.k8s.io/cluster-api/util/predicates" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/source" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" - "github.com/kubesphere/kubekey/v3/pkg" - "github.com/kubesphere/kubekey/v3/pkg/scope" - "github.com/kubesphere/kubekey/v3/util" -) - -var ( - // kkMachineKind contains the schema.GroupVersionKind for the KKMachine type. - kkMachineKind = infrav1.GroupVersion.WithKind("KKMachine") -) - -// InstanceIDIndex defines the kk machine controller's instance ID index. -const ( - InstanceIDIndex = ".spec.instanceID" -) - -// Reconciler reconciles a KKMachine object -type Reconciler struct { - client.Client - mutex sync.Mutex - Scheme *runtime.Scheme - Recorder record.EventRecorder - Tracker *remote.ClusterCacheTracker - WatchFilterValue string - DataDir string -} - -// SetupWithManager sets up the controller with the Manager. -func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { - log := ctrl.LoggerFrom(ctx) - - c, err := ctrl.NewControllerManagedBy(mgr). - WithOptions(options). - For(&infrav1.KKMachine{}). - Watches( - &source.Kind{Type: &clusterv1.Machine{}}, - handler.EnqueueRequestsFromMapFunc(cutil.MachineToInfrastructureMapFunc(infrav1.GroupVersion.WithKind("KKMachine"))), - ). - Watches( - &source.Kind{Type: &infrav1.KKCluster{}}, - handler.EnqueueRequestsFromMapFunc(r.KKClusterToKKMachines(log)), - ). - WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(log, r.WatchFilterValue)). - Build(r) - if err != nil { - return err - } - - // Add index to KKMachine to find by providerID - if err := mgr.GetFieldIndexer().IndexField(ctx, &infrav1.KKMachine{}, - InstanceIDIndex, - r.indexKKMachineByInstanceID, - ); err != nil { - return errors.Wrap(err, "error setting index fields") - } - - requeueKKMachinesForUnpausedCluster := r.requeueKKMachinesForUnpausedCluster(log) - return c.Watch( - &source.Kind{Type: &clusterv1.Cluster{}}, - handler.EnqueueRequestsFromMapFunc(requeueKKMachinesForUnpausedCluster), - predicates.ClusterUnpausedAndInfrastructureReady(log), - ) -} - -//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=kkmachines;kkmachines/status;kkmachines/finalizers,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines;machines/status,verbs=get;list;watch -//+kubebuilder:rbac:groups="",resources=secrets;,verbs=get;list;watch -//+kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch -//+kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;update;patch - -func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, retErr error) { - log := ctrl.LoggerFrom(ctx) - - // Fetch the KKMachine. - kkMachine := &infrav1.KKMachine{} - err := r.Get(ctx, req.NamespacedName, kkMachine) - if err != nil { - if apierrors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - - // Fetch the Machine. - machine, err := cutil.GetOwnerMachine(ctx, r.Client, kkMachine.ObjectMeta) - if err != nil { - return ctrl.Result{}, err - } - if machine == nil { - log.Info("Machine Controller has not yet set OwnerRef") - return ctrl.Result{}, nil - } - - log = log.WithValues("machine", machine.Name) - - // Fetch the Cluster. - cluster, err := cutil.GetClusterFromMetadata(ctx, r.Client, machine.ObjectMeta) - if err != nil { - log.Info("Machine is missing cluster label or cluster does not exist") - return ctrl.Result{}, nil - } - - if annotations.IsPaused(cluster, kkMachine) { - log.Info("KKMachine or linked Cluster is marked as paused. Won't reconcile") - return ctrl.Result{}, nil - } - - log = log.WithValues("cluster", cluster.Name) - - infraCluster, err := util.GetInfraCluster(ctx, r.Client, log, cluster, "kkmachine", r.DataDir) - if err != nil { - return ctrl.Result{}, errors.Wrapf(err, "error getting infra provider cluster object") - } - if infraCluster == nil { - log.Info("KKCluster is not ready yet") - return ctrl.Result{}, nil - } - - // Create the machine scope - machineScope, err := scope.NewMachineScope(scope.MachineScopeParams{ - Client: r.Client, - Logger: &log, - Cluster: cluster, - Machine: machine, - InfraCluster: infraCluster, - KKMachine: kkMachine, - }) - if err != nil { - log.Error(err, "failed to create scope") - return ctrl.Result{}, err - } - - // Always close the scope when exiting this function, so we can persist any KKMachine changes. - defer func() { - if err := machineScope.Close(); err != nil && retErr == nil { - log.Error(err, "failed to patch object") - retErr = err - } - }() - - if !kkMachine.ObjectMeta.DeletionTimestamp.IsZero() { - return r.reconcileDelete(ctx, machineScope) - } - - return r.reconcileNormal(ctx, machineScope, infraCluster, infraCluster) -} - -func (r *Reconciler) reconcileDelete(ctx context.Context, machineScope *scope.MachineScope) (ctrl.Result, error) { - machineScope.Info("Reconcile KKMachine delete") - - // Set the InstanceReadyCondition and patch the object before the blocking operation - conditions.MarkFalse(machineScope.KKMachine, infrav1.InstanceReadyCondition, clusterv1.DeletingReason, clusterv1.ConditionSeverityInfo, "") - if err := machineScope.PatchObject(); err != nil { - machineScope.Error(err, "failed to patch object") - return ctrl.Result{}, err - } - - instance, err := r.reconcileDeleteKKInstance(ctx, machineScope) - if instance != nil || err != nil { - return ctrl.Result{RequeueAfter: 5 * time.Second}, err - } - - machineScope.V(4).Info("Unable to locate KubeKey instance by ID") - - conditions.MarkFalse(machineScope.KKMachine, infrav1.InstanceReadyCondition, clusterv1.DeletedReason, clusterv1.ConditionSeverityInfo, "") - controllerutil.RemoveFinalizer(machineScope.KKMachine, infrav1.MachineFinalizer) - return ctrl.Result{}, nil -} - -func (r *Reconciler) reconcileDeleteKKInstance(ctx context.Context, machineScope *scope.MachineScope) (*infrav1.KKInstance, error) { - // Find existing instance - instance, err := r.findInstance(ctx, machineScope) - if err != nil && !apierrors.IsNotFound(err) { - machineScope.Error(err, "unable to find instance") - return nil, err - } - - if instance != nil { - machineScope.V(3).Info("KubeKey instance found matching deleted KKMachine", "instance", instance.Name) - if conditions.IsTrue(instance, infrav1.KKInstanceDeletingBootstrapCondition) { - machineScope.V(5).Info("instance has not been cleaned up yet", "instance", instance.Name) - return instance, errors.Errorf("kkinstance %s has not been cleaned up yet", instance.Name) - } - - if err := r.deleteInstance(ctx, instance); err != nil { - conditions.MarkFalse(machineScope.KKMachine, infrav1.InstanceReadyCondition, "DeletingFailed", clusterv1.ConditionSeverityWarning, err.Error()) - r.Recorder.Eventf(machineScope.KKMachine, corev1.EventTypeWarning, "FailedDelete", "Failed to delete instance %q: %v", instance.Name, err) - return instance, err - } - r.Recorder.Eventf(machineScope.KKMachine, corev1.EventTypeNormal, "SuccessfulCleaned", "Clean instance %q", instance.Name) - } - return instance, nil -} - -func (r *Reconciler) reconcileNormal(ctx context.Context, machineScope *scope.MachineScope, _ pkg.ClusterScoper, - kkInstanceScope scope.KKInstanceScope) (ctrl.Result, error) { - machineScope.Info("Reconcile KKMachine normal") - - // If the KKMachine is in an error state, return early. - if machineScope.HasFailed() { - machineScope.Info("Error state detected, skipping reconciliation") - return ctrl.Result{}, nil - } - - if machineScope.KKMachine.Labels == nil { - machineScope.KKMachine.Labels = make(map[string]string) - } - - machineScope.KKMachine.Labels[infrav1.KKClusterLabelName] = machineScope.InfraCluster.InfraClusterName() - - if !machineScope.Cluster.Status.InfrastructureReady { - machineScope.Info("Cluster infrastructure is not ready yet") - conditions.MarkFalse(machineScope.KKMachine, infrav1.InstanceReadyCondition, infrav1.WaitingForClusterInfrastructureReason, clusterv1.ConditionSeverityInfo, "") - return ctrl.Result{}, nil - } - - // Make sure bootstrap data is available and populated. - if machineScope.Machine.Spec.Bootstrap.DataSecretName == nil { - machineScope.Info("Bootstrap data secret reference is not yet available") - conditions.MarkFalse(machineScope.KKMachine, infrav1.InstanceReadyCondition, infrav1.WaitingForBootstrapDataReason, clusterv1.ConditionSeverityInfo, "") - return ctrl.Result{}, nil - } - - // Find existing instance - instance, err := r.findInstance(ctx, machineScope) - if err != nil { - machineScope.Error(err, "unable to find instance") - conditions.MarkUnknown(machineScope.KKMachine, infrav1.InstanceReadyCondition, infrav1.InstanceNotFoundReason, err.Error()) - return ctrl.Result{}, err - } - - // If the KKMachine doesn't have our finalizer, add it. - if controllerutil.AddFinalizer(machineScope.KKMachine, infrav1.MachineFinalizer) { - // Register the finalizer after first read operation from KK to avoid orphaning KK resources on delete - if err := machineScope.PatchObject(); err != nil { - machineScope.Error(err, "unable to patch object") - return ctrl.Result{}, err - } - } - - // Create new instance from KKCluster since providerId is nils. - if instance == nil { - // Avoid a flickering condition between InstanceBootstrapStarted and InstanceBootstrapFailed if there's a persistent failure with createInstance - if conditions.GetReason(machineScope.KKMachine, infrav1.InstanceReadyCondition) != infrav1.InstanceBootstrapFailedReason { - conditions.MarkFalse(machineScope.KKMachine, infrav1.InstanceReadyCondition, infrav1.InstanceBootstrapStartedReason, clusterv1.ConditionSeverityInfo, "") - if patchErr := machineScope.PatchObject(); err != nil { - machineScope.Error(patchErr, "failed to patch conditions") - return ctrl.Result{}, patchErr - } - } - - if !r.mutex.TryLock() { - machineScope.V(4).Info("Waiting for the last KKInstance to be created") - return ctrl.Result{RequeueAfter: 2 * time.Second}, nil - } - instance, err = r.createInstance(ctx, machineScope, kkInstanceScope) - r.mutex.Unlock() - if err != nil { - machineScope.Error(err, "unable to create kkInstance") - r.Recorder.Eventf(machineScope.KKMachine, corev1.EventTypeWarning, "FailedCreate", "Failed to create kkInstance: %v", err) - conditions.MarkFalse(machineScope.KKMachine, infrav1.InstanceReadyCondition, infrav1.InstanceBootstrapFailedReason, - clusterv1.ConditionSeverityError, err.Error()) - return ctrl.Result{}, err - } - } - - // Make sure Spec.ProviderID and Spec.InstanceID are always set. - machineScope.SetProviderID(instance.Name, machineScope.Cluster.Name) - machineScope.SetInstanceID(instance.Name) - - existingInstanceState := machineScope.GetInstanceState() - machineScope.SetInstanceState(instance.Status.State) - - // Proceed to reconcile the KKMachine state. - if existingInstanceState == nil || *existingInstanceState != instance.Status.State { - machineScope.Info("KubeKey instance state changed", "state", instance.Status.State, "instance-id", *machineScope.GetInstanceID()) - } - - switch instance.Status.State { - case "": - machineScope.SetNotReady() - conditions.MarkFalse(machineScope.KKMachine, infrav1.InstanceReadyCondition, infrav1.InstanceNotReadyReason, clusterv1.ConditionSeverityWarning, "") - return ctrl.Result{Requeue: true}, nil - case infrav1.InstanceStatePending: - machineScope.SetNotReady() - conditions.MarkFalse(machineScope.KKMachine, infrav1.InstanceReadyCondition, infrav1.InstanceNotReadyReason, clusterv1.ConditionSeverityWarning, "") - return ctrl.Result{Requeue: true}, nil - case infrav1.InstanceStateBootstrapping: - machineScope.SetNotReady() - conditions.MarkFalse(machineScope.KKMachine, infrav1.InstanceReadyCondition, infrav1.InstanceNotReadyReason, clusterv1.ConditionSeverityWarning, "") - return ctrl.Result{Requeue: true}, nil - case infrav1.InstanceStateCleaning, infrav1.InstanceStateCleaned: - machineScope.SetNotReady() - conditions.MarkFalse(machineScope.KKMachine, infrav1.InstanceReadyCondition, infrav1.InstanceCleanedReason, clusterv1.ConditionSeverityWarning, "") - return ctrl.Result{Requeue: true}, nil - case infrav1.InstanceStateRunning: - if err := r.setNodeProviderID(ctx, machineScope, instance); err != nil { - machineScope.Error(err, "failed to patch the Kubernetes node with the machine providerID") - return ctrl.Result{RequeueAfter: 5 * time.Second}, nil - } - - var addresses []clusterv1.MachineAddress - privateIPAddress := clusterv1.MachineAddress{ - Type: clusterv1.MachineInternalIP, - Address: instance.Spec.InternalAddress, - } - addresses = append(addresses, privateIPAddress) - machineScope.SetAddresses(addresses) - - machineScope.SetReady() - conditions.MarkTrue(machineScope.KKMachine, infrav1.InstanceReadyCondition) - case infrav1.InstanceStateInPlaceUpgrading: - machineScope.SetNotReady() - conditions.MarkFalse(machineScope.KKMachine, infrav1.InstanceReadyCondition, infrav1.InstanceInPlaceUpgradingReason, clusterv1.ConditionSeverityWarning, "") - return ctrl.Result{RequeueAfter: 15 * time.Second}, nil - default: - machineScope.SetNotReady() - machineScope.Info("KubeKey instance state is undefined", "state", instance.Status.State, "instance-id", *machineScope.GetInstanceID()) - r.Recorder.Eventf(machineScope.KKMachine, corev1.EventTypeWarning, "InstanceUnhandledState", "KubeKey instance state is undefined") - machineScope.SetFailureReason(capierrors.UpdateMachineError) - machineScope.SetFailureMessage(errors.Errorf("KubeKey instance state %q is undefined", instance.Status.State)) - conditions.MarkUnknown(machineScope.KKMachine, infrav1.InstanceReadyCondition, "", "") - } - - return ctrl.Result{}, nil -} - -func (r *Reconciler) findInstance(ctx context.Context, machineScope *scope.MachineScope) (*infrav1.KKInstance, error) { - machineScope.V(4).Info("Find KubeKey instance") - - kkInstance := &infrav1.KKInstance{} - - // Parse the ProviderID. - pid, err := noderefutil.NewProviderID(machineScope.GetProviderID()) - if err != nil { - if errors.Is(err, noderefutil.ErrEmptyProviderID) { - machineScope.Info("KKMachine does not have an instance id") - return nil, nil - } - return nil, errors.Wrapf(err, "failed to parse Spec.ProviderID") - } - - machineScope.V(4).Info("KKMachine has an instance id", "instance-id", pid.ID()) - // If the ProviderID is populated, describe the instance using the ID. - id := pointer.String(pid.ID()) - - obj := client.ObjectKey{ - Namespace: machineScope.KKMachine.Namespace, - Name: *id, - } - if err := r.Client.Get(ctx, obj, kkInstance); err != nil { - return nil, err - } - // The only case where the instance is nil here is when the providerId is empty and instance could not be found by tags. - return kkInstance, nil -} - -func (r *Reconciler) indexKKMachineByInstanceID(o client.Object) []string { - kkMachine, ok := o.(*infrav1.KKMachine) - if !ok { - return nil - } - - if kkMachine.Spec.InstanceID != nil { - return []string{*kkMachine.Spec.InstanceID} - } - - return nil -} - -// KKClusterToKKMachines is a handler.ToRequestsFunc to be used to enqeue requests for reconciliation -// of KKMachines. -func (r *Reconciler) KKClusterToKKMachines(log logr.Logger) handler.MapFunc { - return func(o client.Object) []ctrl.Request { - c, ok := o.(*infrav1.KKCluster) - if !ok { - panic(fmt.Sprintf("Expected a KKCluster but got a %T", o)) - } - - log := log.WithValues("objectMapper", "kkClusterToKKMachine", "namespace", c.Namespace, "kkCluster", c.Name) - - // Don't handle deleted KKClusters - if !c.ObjectMeta.DeletionTimestamp.IsZero() { - log.V(4).Info("KKCluster has a deletion timestamp, skipping mapping.") - return nil - } - - cluster, err := cutil.GetOwnerCluster(context.TODO(), r.Client, c.ObjectMeta) - switch { - case apierrors.IsNotFound(err) || cluster == nil: - log.V(4).Info("Cluster for KKCluster not found, skipping mapping.") - return nil - case err != nil: - log.Error(err, "Failed to get owning cluster, skipping mapping.") - return nil - } - - return r.requestsForCluster(log, cluster.Namespace, cluster.Name) - } -} - -func (r *Reconciler) requeueKKMachinesForUnpausedCluster(log logr.Logger) handler.MapFunc { - return func(o client.Object) []ctrl.Request { - c, ok := o.(*clusterv1.Cluster) - if !ok { - panic(fmt.Sprintf("Expected a Cluster but got a %T", o)) - } - - log := log.WithValues("objectMapper", "clusterToKKMachine", "namespace", c.Namespace, "cluster", c.Name) - - // Don't handle deleted clusters - if !c.ObjectMeta.DeletionTimestamp.IsZero() { - log.V(4).Info("Cluster has a deletion timestamp, skipping mapping.") - return nil - } - - return r.requestsForCluster(log, c.Namespace, c.Name) - } -} - -func (r *Reconciler) requestsForCluster(log logr.Logger, namespace, name string) []ctrl.Request { - labels := map[string]string{clusterv1.ClusterLabelName: name} - machineList := &clusterv1.MachineList{} - if err := r.Client.List(context.TODO(), machineList, client.InNamespace(namespace), client.MatchingLabels(labels)); err != nil { - log.Error(err, "Failed to get owned Machines, skipping mapping.") - return nil - } - - result := make([]ctrl.Request, 0, len(machineList.Items)) - for _, m := range machineList.Items { - log.WithValues("machine", m.Name) - if m.Spec.InfrastructureRef.GroupVersionKind().Kind != "KKMachine" { - log.V(4).Info("Machine has an InfrastructureRef for a different type, will not add to reconciliation request.") - continue - } - if m.Spec.InfrastructureRef.Name == "" { - log.V(4).Info("Machine has an InfrastructureRef with an empty name, will not add to reconciliation request.") - continue - } - log.WithValues("kkMachine", m.Spec.InfrastructureRef.Name) - log.V(4).Info("Adding KKMachine to reconciliation request.") - result = append(result, ctrl.Request{NamespacedName: client.ObjectKey{Namespace: m.Namespace, Name: m.Spec.InfrastructureRef.Name}}) - } - return result -} diff --git a/controlplane/k3s/PROJECT b/controlplane/k3s/PROJECT deleted file mode 100644 index 76d26ac1..00000000 --- a/controlplane/k3s/PROJECT +++ /dev/null @@ -1,36 +0,0 @@ -domain: cluster.x-k8s.io -layout: -- go.kubebuilder.io/v3 -plugins: - manifests.sdk.operatorframework.io/v2: {} - scorecard.sdk.operatorframework.io/v2: {} -projectName: cluster-api-control-plane-provider-kubekey-k3s -repo: github.com/kubesphere/kubekey/controlplane/k3s -resources: -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: cluster.x-k8s.io - group: controlplane - kind: K3sControlPlane - path: github.com/kubesphere/kubekey/controlplane/k3s/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: cluster.x-k8s.io - group: controlplane - kind: K3sControlPlaneTemplate - path: github.com/kubesphere/kubekey/controlplane/k3s/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -version: "3" diff --git a/controlplane/k3s/api/v1beta1/condition_consts.go b/controlplane/k3s/api/v1beta1/condition_consts.go deleted file mode 100644 index b220c55c..00000000 --- a/controlplane/k3s/api/v1beta1/condition_consts.go +++ /dev/null @@ -1,135 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package v1beta1 - -import ( - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" -) - -// Conditions and condition Reasons for the K3sControlPlane object. - -const ( - // MachinesReadyCondition reports an aggregate of current status of the machines controlled by the K3sControlPlane. - MachinesReadyCondition clusterv1.ConditionType = "MachinesReady" -) - -const ( - // CertificatesAvailableCondition documents that cluster certificates were generated as part of the - // processing of a a K3sControlPlane object. - CertificatesAvailableCondition clusterv1.ConditionType = "CertificatesAvailable" - - // CertificatesGenerationFailedReason (Severity=Warning) documents a K3sControlPlane controller detecting - // an error while generating certificates; those kind of errors are usually temporary and the controller - // automatically recover from them. - CertificatesGenerationFailedReason = "CertificatesGenerationFailed" -) - -const ( - // AvailableCondition documents that the first control plane instance has completed the kubeadm init operation - // and so the control plane is available and an API server instance is ready for processing requests. - AvailableCondition clusterv1.ConditionType = "Available" - - // WaitingForKubeadmInitReason (Severity=Info) documents a K3sControlPlane object waiting for the first - // control plane instance to complete the kubeadm init operation. - WaitingForKubeadmInitReason = "WaitingForKubeadmInit" -) - -const ( - // MachinesSpecUpToDateCondition documents that the spec of the machines controlled by the K3sControlPlane - // is up to date. When this condition is false, the K3sControlPlane is executing a rolling upgrade. - MachinesSpecUpToDateCondition clusterv1.ConditionType = "MachinesSpecUpToDate" - - // RollingUpdateInProgressReason (Severity=Warning) documents a K3sControlPlane object executing a - // rolling upgrade for aligning the machines spec to the desired state. - RollingUpdateInProgressReason = "RollingUpdateInProgress" -) - -const ( - // ResizedCondition documents a K3sControlPlane that is resizing the set of controlled machines. - ResizedCondition clusterv1.ConditionType = "Resized" - - // ScalingUpReason (Severity=Info) documents a K3sControlPlane that is increasing the number of replicas. - ScalingUpReason = "ScalingUp" - - // ScalingDownReason (Severity=Info) documents a K3sControlPlane that is decreasing the number of replicas. - ScalingDownReason = "ScalingDown" -) - -const ( - // ControlPlaneComponentsHealthyCondition reports the overall status of control plane components - // implemented as static pods generated by kubeadm including kube-api-server, kube-controller manager, - // kube-scheduler and etcd if managed. - ControlPlaneComponentsHealthyCondition clusterv1.ConditionType = "ControlPlaneComponentsHealthy" - - // ControlPlaneComponentsUnhealthyReason (Severity=Error) documents a control plane component not healthy. - ControlPlaneComponentsUnhealthyReason = "ControlPlaneComponentsUnhealthy" - - // ControlPlaneComponentsUnknownReason reports a control plane component in unknown status. - ControlPlaneComponentsUnknownReason = "ControlPlaneComponentsUnknown" - - // ControlPlaneComponentsInspectionFailedReason documents a failure in inspecting the control plane component status. - ControlPlaneComponentsInspectionFailedReason = "ControlPlaneComponentsInspectionFailed" - - // MachineAgentHealthyCondition reports a machine's agent operational status. - // NOTE: This conditions exists only if a stacked etcd cluster is used. - MachineAgentHealthyCondition clusterv1.ConditionType = "AgentHealthy" - - // PodProvisioningReason (Severity=Info) documents a pod waiting to be provisioned i.e., Pod is in "Pending" phase. - PodProvisioningReason = "PodProvisioning" - - // PodMissingReason (Severity=Error) documents a pod does not exist. - PodMissingReason = "PodMissing" - - // PodFailedReason (Severity=Error) documents if a pod failed during provisioning i.e., e.g CrashLoopbackOff, ImagePullBackOff - // or if all the containers in a pod have terminated. - PodFailedReason = "PodFailed" - - // PodInspectionFailedReason documents a failure in inspecting the pod status. - PodInspectionFailedReason = "PodInspectionFailed" -) - -const ( - // EtcdClusterHealthyCondition documents the overall etcd cluster's health. - EtcdClusterHealthyCondition clusterv1.ConditionType = "EtcdClusterHealthyCondition" - - // EtcdClusterInspectionFailedReason documents a failure in inspecting the etcd cluster status. - EtcdClusterInspectionFailedReason = "EtcdClusterInspectionFailed" - - // MachineEtcdMemberHealthyCondition report the machine's etcd member's health status. - // NOTE: This conditions exists only if a stacked etcd cluster is used. - MachineEtcdMemberHealthyCondition clusterv1.ConditionType = "EtcdMemberHealthy" - - // EtcdMemberInspectionFailedReason documents a failure in inspecting the etcd member status. - EtcdMemberInspectionFailedReason = "MemberInspectionFailed" - - // MachinesCreatedCondition documents that the machines controlled by the K3sControlPlane are created. - // When this condition is false, it indicates that there was an error when cloning the infrastructure/bootstrap template or - // when generating the machine object. - MachinesCreatedCondition clusterv1.ConditionType = "MachinesCreated" - - // InfrastructureTemplateCloningFailedReason (Severity=Error) documents a K3sControlPlane failing to - // clone the infrastructure template. - InfrastructureTemplateCloningFailedReason = "InfrastructureTemplateCloningFailed" - - // BootstrapTemplateCloningFailedReason (Severity=Error) documents a K3sControlPlane failing to - // clone the bootstrap template. - BootstrapTemplateCloningFailedReason = "BootstrapTemplateCloningFailed" - - // MachineGenerationFailedReason (Severity=Error) documents a K3sControlPlane failing to - // generate a machine object. - MachineGenerationFailedReason = "MachineGenerationFailed" -) diff --git a/controlplane/k3s/api/v1beta1/groupversion_info.go b/controlplane/k3s/api/v1beta1/groupversion_info.go deleted file mode 100644 index 2e71079c..00000000 --- a/controlplane/k3s/api/v1beta1/groupversion_info.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package v1beta1 contains API Schema definitions for the controlplane v1beta1 API group -// +kubebuilder:object:generate=true -// +groupName=controlplane.cluster.x-k8s.io -package v1beta1 - -import ( - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/scheme" -) - -var ( - // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "controlplane.cluster.x-k8s.io", Version: "v1beta1"} - - // SchemeBuilder is used to add go types to the GroupVersionKind scheme - SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} - - // AddToScheme adds the types in this group-version to the given scheme. - AddToScheme = SchemeBuilder.AddToScheme -) diff --git a/controlplane/k3s/api/v1beta1/k3scontrolplane_types.go b/controlplane/k3s/api/v1beta1/k3scontrolplane_types.go deleted file mode 100644 index 264fc73e..00000000 --- a/controlplane/k3s/api/v1beta1/k3scontrolplane_types.go +++ /dev/null @@ -1,253 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/errors" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" -) - -// RolloutStrategyType defines the rollout strategies for a K3sControlPlane. -type RolloutStrategyType string - -const ( - // RollingUpdateStrategyType replaces the old control planes by new one using rolling update - // i.e. gradually scale up or down the old control planes and scale up or down the new one. - RollingUpdateStrategyType RolloutStrategyType = "RollingUpdate" -) - -const ( - // K3sControlPlaneFinalizer is the finalizer applied to K3sControlPlane resources - // by its managing controller. - K3sControlPlaneFinalizer = "k3s.controlplane.cluster.x-k8s.io" - - // SkipCoreDNSAnnotation annotation explicitly skips reconciling CoreDNS if set. - SkipCoreDNSAnnotation = "controlplane.cluster.x-k8s.io/skip-coredns" - - // SkipKubeProxyAnnotation annotation explicitly skips reconciling kube-proxy if set. - SkipKubeProxyAnnotation = "controlplane.cluster.x-k8s.io/skip-kube-proxy" - - // K3sServerConfigurationAnnotation is a machine annotation that stores the json-marshalled string of K3SCP ClusterConfiguration. - // This annotation is used to detect any changes in ClusterConfiguration and trigger machine rollout in K3SCP. - K3sServerConfigurationAnnotation = "controlplane.cluster.x-k8s.io/k3s-server-configuration" -) - -// K3sControlPlaneSpec defines the desired state of K3sControlPlane -type K3sControlPlaneSpec struct { - // Number of desired machines. Defaults to 1. When stacked etcd is used only - // odd numbers are permitted, as per [etcd best practice](https://etcd.io/docs/v3.3.12/faq/#why-an-odd-number-of-cluster-members). - // This is a pointer to distinguish between explicit zero and not specified. - // +optional - Replicas *int32 `json:"replicas,omitempty"` - - // Version defines the desired K3s version. - Version string `json:"version"` - - // MachineTemplate contains information about how machines - // should be shaped when creating or updating a control plane. - MachineTemplate K3sControlPlaneMachineTemplate `json:"machineTemplate"` - - // K3sConfigSpec is a K3sConfigSpec - // to use for initializing and joining machines to the control plane. - // +optional - K3sConfigSpec infrabootstrapv1.K3sConfigSpec `json:"k3sConfigSpec,omitempty"` - - // RolloutAfter is a field to indicate a rollout should be performed - // after the specified time even if no changes have been made to the - // K3sControlPlane. - // - // +optional - RolloutAfter *metav1.Time `json:"rolloutAfter,omitempty"` - - // The RolloutStrategy to use to replace control plane machines with - // new ones. - // +optional - // +kubebuilder:default={type: "RollingUpdate", rollingUpdate: {maxSurge: 1}} - RolloutStrategy *RolloutStrategy `json:"rolloutStrategy,omitempty"` -} - -// K3sControlPlaneMachineTemplate defines the template for Machines -// in a K3sControlPlane object. -type K3sControlPlaneMachineTemplate struct { - // Standard object's metadata. - // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - // +optional - ObjectMeta clusterv1.ObjectMeta `json:"metadata,omitempty"` - - // InfrastructureRef is a required reference to a custom resource - // offered by an infrastructure provider. - InfrastructureRef corev1.ObjectReference `json:"infrastructureRef"` - - // NodeDrainTimeout is the total amount of time that the controller will spend on draining a controlplane node - // The default value is 0, meaning that the node can be drained without any time limitations. - // NOTE: NodeDrainTimeout is different from `kubectl drain --timeout` - // +optional - NodeDrainTimeout *metav1.Duration `json:"nodeDrainTimeout,omitempty"` - - // NodeDeletionTimeout defines how long the machine controller will attempt to delete the Node that the Machine - // hosts after the Machine is marked for deletion. A duration of 0 will retry deletion indefinitely. - // If no value is provided, the default value for this property of the Machine resource will be used. - // +optional - NodeDeletionTimeout *metav1.Duration `json:"nodeDeletionTimeout,omitempty"` -} - -// RolloutStrategy describes how to replace existing machines -// with new ones. -type RolloutStrategy struct { - // Type of rollout. Currently the only supported strategy is - // "RollingUpdate". - // Default is RollingUpdate. - // +optional - Type RolloutStrategyType `json:"type,omitempty"` - - // Rolling update config params. Present only if - // RolloutStrategyType = RollingUpdate. - // +optional - RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"` -} - -// RollingUpdate is used to control the desired behavior of rolling update. -type RollingUpdate struct { - // The maximum number of control planes that can be scheduled above or under the - // desired number of control planes. - // Value can be an absolute number 1 or 0. - // Defaults to 1. - // Example: when this is set to 1, the control plane can be scaled - // up immediately when the rolling update starts. - // +optional - MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"` -} - -// K3sControlPlaneStatus defines the observed state of K3sControlPlane -type K3sControlPlaneStatus struct { - // Selector is the label selector in string format to avoid introspection - // by clients, and is used to provide the CRD-based integration for the - // scale subresource and additional integrations for things like kubectl - // describe.. The string will be in the same format as the query-param syntax. - // More info about label selectors: http://kubernetes.io/docs/user-guide/labels#label-selectors - // +optional - Selector string `json:"selector,omitempty"` - - // Total number of non-terminated machines targeted by this control plane - // (their labels match the selector). - // +optional - Replicas int32 `json:"replicas"` - - // Version represents the minimum Kubernetes version for the control plane machines - // in the cluster. - // +optional - Version *string `json:"version,omitempty"` - - // Total number of non-terminated machines targeted by this control plane - // that have the desired template spec. - // +optional - UpdatedReplicas int32 `json:"updatedReplicas"` - - // Total number of fully running and ready control plane machines. - // +optional - ReadyReplicas int32 `json:"readyReplicas"` - - // Total number of unavailable machines targeted by this control plane. - // This is the total number of machines that are still required for - // the deployment to have 100% available capacity. They may either - // be machines that are running but not yet ready or machines - // that still have not been created. - // +optional - UnavailableReplicas int32 `json:"unavailableReplicas"` - - // Initialized denotes whether or not the control plane has the - // uploaded kubeadm-config configmap. - // +optional - Initialized bool `json:"initialized"` - - // Ready denotes that the K3sControlPlane API Server is ready to - // receive requests. - // +optional - Ready bool `json:"ready"` - - // FailureReason indicates that there is a terminal problem reconciling the - // state, and will be set to a token value suitable for - // programmatic interpretation. - // +optional - FailureReason errors.KubeadmControlPlaneStatusError `json:"failureReason,omitempty"` - - // ErrorMessage indicates that there is a terminal problem reconciling the - // state, and will be set to a descriptive error message. - // +optional - FailureMessage *string `json:"failureMessage,omitempty"` - - // ObservedGeneration is the latest generation observed by the controller. - // +optional - ObservedGeneration int64 `json:"observedGeneration,omitempty"` - - // Conditions defines current service state of the K3sControlPlane. - // +optional - Conditions clusterv1.Conditions `json:"conditions,omitempty"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:resource:path=k3scontrolplanes,shortName=k3scp,scope=Namespaced,categories=cluster-api -// +kubebuilder:storageversion -// +kubebuilder:subresource:status -// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.selector -// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels['cluster\\.x-k8s\\.io/cluster-name']",description="Cluster" -// +kubebuilder:printcolumn:name="Initialized",type=boolean,JSONPath=".status.initialized",description="This denotes whether or not the control plane has the uploaded k3s-config configmap" -// +kubebuilder:printcolumn:name="API Server Available",type=boolean,JSONPath=".status.ready",description="K3sControlPlane API Server is ready to receive requests" -// +kubebuilder:printcolumn:name="Desired",type=integer,JSONPath=".spec.replicas",description="Total number of machines desired by this control plane",priority=10 -// +kubebuilder:printcolumn:name="Replicas",type=integer,JSONPath=".status.replicas",description="Total number of non-terminated machines targeted by this control plane" -// +kubebuilder:printcolumn:name="Ready",type=integer,JSONPath=".status.readyReplicas",description="Total number of fully running and ready control plane machines" -// +kubebuilder:printcolumn:name="Updated",type=integer,JSONPath=".status.updatedReplicas",description="Total number of non-terminated machines targeted by this control plane that have the desired template spec" -// +kubebuilder:printcolumn:name="Unavailable",type=integer,JSONPath=".status.unavailableReplicas",description="Total number of unavailable machines targeted by this control plane" -// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of KubeadmControlPlane" -// +kubebuilder:printcolumn:name="Version",type=string,JSONPath=".spec.version",description="Kubernetes version associated with this control plane" - -// K3sControlPlane is the Schema for the k3scontrolplanes API -type K3sControlPlane struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec K3sControlPlaneSpec `json:"spec,omitempty"` - Status K3sControlPlaneStatus `json:"status,omitempty"` -} - -// GetConditions returns the set of conditions for this object. -func (in *K3sControlPlane) GetConditions() clusterv1.Conditions { - return in.Status.Conditions -} - -// SetConditions sets the conditions on this object. -func (in *K3sControlPlane) SetConditions(conditions clusterv1.Conditions) { - in.Status.Conditions = conditions -} - -//+kubebuilder:object:root=true - -// K3sControlPlaneList contains a list of K3sControlPlane -type K3sControlPlaneList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []K3sControlPlane `json:"items"` -} - -func init() { - SchemeBuilder.Register(&K3sControlPlane{}, &K3sControlPlaneList{}) -} diff --git a/controlplane/k3s/api/v1beta1/k3scontrolplane_webhook.go b/controlplane/k3s/api/v1beta1/k3scontrolplane_webhook.go deleted file mode 100644 index f3e0ad1a..00000000 --- a/controlplane/k3s/api/v1beta1/k3scontrolplane_webhook.go +++ /dev/null @@ -1,472 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "encoding/json" - "fmt" - "strings" - - "github.com/blang/semver" - jsonpatch "github.com/evanphx/json-patch" - "github.com/pkg/errors" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/utils/pointer" - "sigs.k8s.io/cluster-api/util/version" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" -) - -func (in *K3sControlPlane) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(in). - Complete() -} - -// +kubebuilder:webhook:verbs=create;update,path=/mutate-controlplane-cluster-x-k8s-io-v1beta1-k3scontrolplane,mutating=true,failurePolicy=fail,matchPolicy=Equivalent,groups=controlplane.cluster.x-k8s.io,resources=k3scontrolplanes,versions=v1beta1,name=default.k3scontrolplane.controlplane.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 -// +kubebuilder:webhook:verbs=create;update,path=/validate-controlplane-cluster-x-k8s-io-v1beta1-k3scontrolplane,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=controlplane.cluster.x-k8s.io,resources=k3scontrolplanes,versions=v1beta1,name=validation.k3scontrolplane.controlplane.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 - -var _ webhook.Defaulter = &K3sControlPlane{} -var _ webhook.Validator = &K3sControlPlane{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (in *K3sControlPlane) Default() { - defaultK3sControlPlaneSpec(&in.Spec, in.Namespace) -} - -func defaultK3sControlPlaneSpec(s *K3sControlPlaneSpec, namespace string) { - if s.Replicas == nil { - replicas := int32(1) - s.Replicas = &replicas - } - - if s.MachineTemplate.InfrastructureRef.Namespace == "" { - s.MachineTemplate.InfrastructureRef.Namespace = namespace - } - - if !strings.HasPrefix(s.Version, "v") { - s.Version = "v" + s.Version - } - - if s.K3sConfigSpec.ServerConfiguration == nil { - s.K3sConfigSpec.ServerConfiguration = &infrabootstrapv1.ServerConfiguration{} - } - - if s.K3sConfigSpec.ServerConfiguration.Database.DataStoreEndPoint == "" && s.K3sConfigSpec.ServerConfiguration.Database.ClusterInit == nil { - s.K3sConfigSpec.ServerConfiguration.Database.ClusterInit = pointer.Bool(true) - } - - infrabootstrapv1.DefaultK3sConfigSpec(&s.K3sConfigSpec) - - s.RolloutStrategy = defaultRolloutStrategy(s.RolloutStrategy) -} - -func defaultRolloutStrategy(rolloutStrategy *RolloutStrategy) *RolloutStrategy { - ios1 := intstr.FromInt(1) - - if rolloutStrategy == nil { - rolloutStrategy = &RolloutStrategy{} - } - - // Enforce RollingUpdate strategy and default MaxSurge if not set. - if rolloutStrategy != nil { - if len(rolloutStrategy.Type) == 0 { - rolloutStrategy.Type = RollingUpdateStrategyType - } - if rolloutStrategy.Type == RollingUpdateStrategyType { - if rolloutStrategy.RollingUpdate == nil { - rolloutStrategy.RollingUpdate = &RollingUpdate{} - } - rolloutStrategy.RollingUpdate.MaxSurge = intstr.ValueOrDefault(rolloutStrategy.RollingUpdate.MaxSurge, ios1) - } - } - - return rolloutStrategy -} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (in *K3sControlPlane) ValidateCreate() error { - spec := in.Spec - allErrs := validateK3sControlPlaneSpec(spec, in.Namespace, field.NewPath("spec")) - allErrs = append(allErrs, validateServerConfiguration(spec.K3sConfigSpec.ServerConfiguration, nil, field.NewPath("spec", "k3sConfigSpec", "serverConfiguration"))...) - allErrs = append(allErrs, spec.K3sConfigSpec.Validate(field.NewPath("spec", "k3sConfigSpec"))...) - if len(allErrs) > 0 { - return apierrors.NewInvalid(GroupVersion.WithKind("K3sControlPlane").GroupKind(), in.Name, allErrs) - } - return nil -} - -const ( - spec = "spec" - k3sConfigSpec = "k3sConfigSpec" - preK3sCommands = "preK3sCommands" - postK3sCommands = "postK3sCommands" - files = "files" -) - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (in *K3sControlPlane) ValidateUpdate(old runtime.Object) error { - // add a * to indicate everything beneath is ok. - // For example, {"spec", "*"} will allow any path under "spec" to change. - allowedPaths := [][]string{ - {"metadata", "*"}, - {spec, k3sConfigSpec, preK3sCommands}, - {spec, k3sConfigSpec, postK3sCommands}, - {spec, k3sConfigSpec, files}, - {spec, "machineTemplate", "metadata", "*"}, - {spec, "machineTemplate", "infrastructureRef", "apiVersion"}, - {spec, "machineTemplate", "infrastructureRef", "name"}, - {spec, "machineTemplate", "infrastructureRef", "kind"}, - {spec, "machineTemplate", "nodeDrainTimeout"}, - {spec, "machineTemplate", "nodeDeletionTimeout"}, - {spec, "replicas"}, - {spec, "version"}, - {spec, "rolloutAfter"}, - {spec, "rolloutStrategy", "*"}, - } - - allErrs := validateK3sControlPlaneSpec(in.Spec, in.Namespace, field.NewPath("spec")) - - prev, ok := old.(*K3sControlPlane) - if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expecting K3sControlPlane but got a %T", old)) - } - - originalJSON, err := json.Marshal(prev) - if err != nil { - return apierrors.NewInternalError(err) - } - modifiedJSON, err := json.Marshal(in) - if err != nil { - return apierrors.NewInternalError(err) - } - - diff, err := jsonpatch.CreateMergePatch(originalJSON, modifiedJSON) - if err != nil { - return apierrors.NewInternalError(err) - } - jsonPatch := map[string]interface{}{} - if err := json.Unmarshal(diff, &jsonPatch); err != nil { - return apierrors.NewInternalError(err) - } - // Build a list of all paths that are trying to change - diffpaths := paths([]string{}, jsonPatch) - // Every path in the diff must be valid for the update function to work. - for _, path := range diffpaths { - // Ignore paths that are empty - if len(path) == 0 { - continue - } - if !allowed(allowedPaths, path) { - if len(path) == 1 { - allErrs = append(allErrs, field.Forbidden(field.NewPath(path[0]), "cannot be modified")) - continue - } - allErrs = append(allErrs, field.Forbidden(field.NewPath(path[0], path[1:]...), "cannot be modified")) - } - } - - allErrs = append(allErrs, in.validateVersion(prev.Spec.Version)...) - allErrs = append(allErrs, validateServerConfiguration(in.Spec.K3sConfigSpec.ServerConfiguration, prev.Spec.K3sConfigSpec.ServerConfiguration, field.NewPath("spec", "k3sConfigSpec", "serverConfiguration"))...) - allErrs = append(allErrs, in.Spec.K3sConfigSpec.Validate(field.NewPath("spec", "K3sConfigSpec"))...) - - if len(allErrs) > 0 { - return apierrors.NewInvalid(GroupVersion.WithKind("K3sControlPlane").GroupKind(), in.Name, allErrs) - } - - return nil -} - -func validateK3sControlPlaneSpec(s K3sControlPlaneSpec, namespace string, pathPrefix *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - - if s.Replicas == nil { - allErrs = append( - allErrs, - field.Required( - pathPrefix.Child("replicas"), - "is required", - ), - ) - } else if *s.Replicas <= 0 { - // The use of the scale subresource should provide a guarantee that negative values - // should not be accepted for this field, but since we have to validate that Replicas != 0 - // it doesn't hurt to also additionally validate for negative numbers here as well. - allErrs = append( - allErrs, - field.Forbidden( - pathPrefix.Child("replicas"), - "cannot be less than or equal to 0", - ), - ) - } - - if s.MachineTemplate.InfrastructureRef.APIVersion == "" { - allErrs = append( - allErrs, - field.Invalid( - pathPrefix.Child("machineTemplate", "infrastructure", "apiVersion"), - s.MachineTemplate.InfrastructureRef.APIVersion, - "cannot be empty", - ), - ) - } - if s.MachineTemplate.InfrastructureRef.Kind == "" { - allErrs = append( - allErrs, - field.Invalid( - pathPrefix.Child("machineTemplate", "infrastructure", "kind"), - s.MachineTemplate.InfrastructureRef.Kind, - "cannot be empty", - ), - ) - } - if s.MachineTemplate.InfrastructureRef.Name == "" { - allErrs = append( - allErrs, - field.Invalid( - pathPrefix.Child("machineTemplate", "infrastructure", "name"), - s.MachineTemplate.InfrastructureRef.Name, - "cannot be empty", - ), - ) - } - if s.MachineTemplate.InfrastructureRef.Namespace != namespace { - allErrs = append( - allErrs, - field.Invalid( - pathPrefix.Child("machineTemplate", "infrastructure", "namespace"), - s.MachineTemplate.InfrastructureRef.Namespace, - "must match metadata.namespace", - ), - ) - } - - if !version.KubeSemver.MatchString(s.Version) { - allErrs = append(allErrs, field.Invalid(pathPrefix.Child("version"), s.Version, "must be a valid semantic version")) - } - - allErrs = append(allErrs, validateRolloutStrategy(s.RolloutStrategy, s.Replicas, pathPrefix.Child("rolloutStrategy"))...) - - return allErrs -} - -func validateRolloutStrategy(rolloutStrategy *RolloutStrategy, replicas *int32, pathPrefix *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - - if rolloutStrategy == nil { - return allErrs - } - - if rolloutStrategy.Type != RollingUpdateStrategyType { - allErrs = append( - allErrs, - field.Required( - pathPrefix.Child("type"), - "only RollingUpdateStrategyType is supported", - ), - ) - } - - ios1 := intstr.FromInt(1) - ios0 := intstr.FromInt(0) - - if *rolloutStrategy.RollingUpdate.MaxSurge == ios0 && (replicas != nil && *replicas < int32(3)) { - allErrs = append( - allErrs, - field.Required( - pathPrefix.Child("rollingUpdate"), - "when K3sControlPlane is configured to scale-in, replica count needs to be at least 3", - ), - ) - } - - if *rolloutStrategy.RollingUpdate.MaxSurge != ios1 && *rolloutStrategy.RollingUpdate.MaxSurge != ios0 { - allErrs = append( - allErrs, - field.Required( - pathPrefix.Child("rollingUpdate", "maxSurge"), - "value must be 1 or 0", - ), - ) - } - - return allErrs -} - -func validateServerConfiguration(newServerConfiguration, oldServerConfiguration *infrabootstrapv1.ServerConfiguration, pathPrefix *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - - if newServerConfiguration == nil { - return allErrs - } - - if newServerConfiguration.Database.ClusterInit != nil { - if *newServerConfiguration.Database.ClusterInit && newServerConfiguration.Database.DataStoreEndPoint != "" { - allErrs = append( - allErrs, - field.Forbidden( - pathPrefix.Child("database", "clusterInit"), - "cannot have both external and local etcd", - ), - ) - } - } - - // update validations - if oldServerConfiguration != nil { - if newServerConfiguration.Database.ClusterInit != nil { - if *newServerConfiguration.Database.ClusterInit && oldServerConfiguration.Database.DataStoreEndPoint != "" { - allErrs = append( - allErrs, - field.Forbidden( - pathPrefix.Child("database", "clusterInit"), - "cannot change between external and local etcd", - ), - ) - } - } - - if oldServerConfiguration.Database.ClusterInit != nil { - if newServerConfiguration.Database.DataStoreEndPoint != "" && *oldServerConfiguration.Database.ClusterInit { - allErrs = append( - allErrs, - field.Forbidden( - pathPrefix.Child("database", "dataStoreEndPoint"), - "cannot change between external and local etcd", - ), - ) - } - } - } - - return allErrs -} - -func allowed(allowList [][]string, path []string) bool { - for _, allowed := range allowList { - if pathsMatch(allowed, path) { - return true - } - } - return false -} - -func pathsMatch(allowed, path []string) bool { - // if either are empty then no match can be made - if len(allowed) == 0 || len(path) == 0 { - return false - } - i := 0 - for i = range path { - // reached the end of the allowed path and no match was found - if i > len(allowed)-1 { - return false - } - if allowed[i] == "*" { - return true - } - if path[i] != allowed[i] { - return false - } - } - // path has been completely iterated and has not matched the end of the path. - // e.g. allowed: []string{"a","b","c"}, path: []string{"a"} - return i >= len(allowed)-1 -} - -// paths builds a slice of paths that are being modified. -func paths(path []string, diff map[string]interface{}) [][]string { - allPaths := [][]string{} - for key, m := range diff { - nested, ok := m.(map[string]interface{}) - if !ok { - // We have to use a copy of path, because otherwise the slice we append to - // allPaths would be overwritten in another iteration. - tmp := make([]string, len(path)) - copy(tmp, path) - allPaths = append(allPaths, append(tmp, key)) - continue - } - allPaths = append(allPaths, paths(append(path, key), nested)...) - } - return allPaths -} - -func (in *K3sControlPlane) validateVersion(previousVersion string) (allErrs field.ErrorList) { - fromVersion, err := version.ParseMajorMinorPatch(previousVersion) - if err != nil { - allErrs = append(allErrs, - field.InternalError( - field.NewPath("spec", "version"), - errors.Wrapf(err, "failed to parse current k3scontrolplane version: %s", previousVersion), - ), - ) - return allErrs - } - - toVersion, err := version.ParseMajorMinorPatch(in.Spec.Version) - if err != nil { - allErrs = append(allErrs, - field.InternalError( - field.NewPath("spec", "version"), - errors.Wrapf(err, "failed to parse updated k3scontrolplane version: %s", in.Spec.Version), - ), - ) - return allErrs - } - - // Check if we're trying to upgrade to Kubernetes v1.19.0, which is not supported. - // - // See https://github.com/kubernetes-sigs/cluster-api/issues/3564 - if fromVersion.NE(toVersion) && toVersion.Equals(semver.MustParse("1.19.0")) { - allErrs = append(allErrs, - field.Forbidden( - field.NewPath("spec", "version"), - "cannot update Kubernetes version to v1.19.0, for more information see https://github.com/kubernetes-sigs/cluster-api/issues/3564", - ), - ) - return allErrs - } - - // Since upgrades to the next minor version are allowed, irrespective of the patch version. - ceilVersion := semver.Version{ - Major: fromVersion.Major, - Minor: fromVersion.Minor + 2, - Patch: 0, - } - if toVersion.GTE(ceilVersion) { - allErrs = append(allErrs, - field.Forbidden( - field.NewPath("spec", "version"), - fmt.Sprintf("cannot update Kubernetes version from %s to %s", previousVersion, in.Spec.Version), - ), - ) - } - - return allErrs -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (in *K3sControlPlane) ValidateDelete() error { - return nil -} diff --git a/controlplane/k3s/api/v1beta1/k3scontrolplanetemplate_types.go b/controlplane/k3s/api/v1beta1/k3scontrolplanetemplate_types.go deleted file mode 100644 index 795b8495..00000000 --- a/controlplane/k3s/api/v1beta1/k3scontrolplanetemplate_types.go +++ /dev/null @@ -1,106 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" -) - -// K3sControlPlaneTemplateSpec defines the desired state of K3sControlPlaneTemplate -type K3sControlPlaneTemplateSpec struct { - Template K3sControlPlaneTemplateResource `json:"template"` -} - -// K3sControlPlaneTemplateResource describes the data needed to create a K3sControlPlane from a template. -type K3sControlPlaneTemplateResource struct { - Spec K3sControlPlaneTemplateResourceSpec `json:"spec"` -} - -// K3sControlPlaneTemplateResourceSpec defines the desired state of K3sControlPlane. -// NOTE: K3sControlPlaneTemplateResourceSpec is similar to K3sControlPlaneSpec but -// omits Replicas and Version fields. These fields do not make sense on the K3sControlPlaneTemplate, -// because they are calculated by the Cluster topology reconciler during reconciliation and thus cannot -// be configured on the K3sControlPlaneTemplate. -type K3sControlPlaneTemplateResourceSpec struct { - // MachineTemplate contains information about how machines - // should be shaped when creating or updating a control plane. - // +optional - MachineTemplate *K3sControlPlaneTemplateMachineTemplate `json:"machineTemplate,omitempty"` - - // K3sConfigSpec is a K3sConfigSpec - // to use for initializing and joining machines to the control plane. - K3sConfigSpec infrabootstrapv1.K3sConfigSpec `json:"k3sConfigSpec"` - - // RolloutAfter is a field to indicate a rollout should be performed - // after the specified time even if no changes have been made to the - // K3sControlPlane. - // - // +optional - RolloutAfter *metav1.Time `json:"rolloutAfter,omitempty"` - - // The RolloutStrategy to use to replace control plane machines with - // new ones. - // +optional - // +kubebuilder:default={type: "RollingUpdate", rollingUpdate: {maxSurge: 1}} - RolloutStrategy *RolloutStrategy `json:"rolloutStrategy,omitempty"` -} - -// K3sControlPlaneTemplateMachineTemplate defines the template for Machines -// in a K3sControlPlaneTemplate object. -// NOTE: K3sControlPlaneTemplateMachineTemplate is similar to K3sControlPlaneMachineTemplate but -// omits ObjectMeta and InfrastructureRef fields. These fields do not make sense on the K3sControlPlaneTemplate, -// because they are calculated by the Cluster topology reconciler during reconciliation and thus cannot -// be configured on the K3sControlPlaneTemplate. -type K3sControlPlaneTemplateMachineTemplate struct { - // NodeDrainTimeout is the total amount of time that the controller will spend on draining a controlplane node - // The default value is 0, meaning that the node can be drained without any time limitations. - // NOTE: NodeDrainTimeout is different from `kubectl drain --timeout` - // +optional - NodeDrainTimeout *metav1.Duration `json:"nodeDrainTimeout,omitempty"` - - // NodeDeletionTimeout defines how long the machine controller will attempt to delete the Node that the Machine - // hosts after the Machine is marked for deletion. A duration of 0 will retry deletion indefinitely. - // If no value is provided, the default value for this property of the Machine resource will be used. - // +optional - NodeDeletionTimeout *metav1.Duration `json:"nodeDeletionTimeout,omitempty"` -} - -//+kubebuilder:object:root=true -//+kubebuilder:subresource:status - -// K3sControlPlaneTemplate is the Schema for the k3scontrolplanetemplates API -type K3sControlPlaneTemplate struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec K3sControlPlaneTemplateSpec `json:"spec,omitempty"` -} - -//+kubebuilder:object:root=true - -// K3sControlPlaneTemplateList contains a list of K3sControlPlaneTemplate -type K3sControlPlaneTemplateList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []K3sControlPlaneTemplate `json:"items"` -} - -func init() { - SchemeBuilder.Register(&K3sControlPlaneTemplate{}, &K3sControlPlaneTemplateList{}) -} diff --git a/controlplane/k3s/api/v1beta1/k3scontrolplanetemplate_webhook.go b/controlplane/k3s/api/v1beta1/k3scontrolplanetemplate_webhook.go deleted file mode 100644 index c52813b3..00000000 --- a/controlplane/k3s/api/v1beta1/k3scontrolplanetemplate_webhook.go +++ /dev/null @@ -1,106 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "fmt" - "reflect" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" - "sigs.k8s.io/cluster-api/feature" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" -) - -const k3sControlPlaneTemplateImmutableMsg = "K3sControlPlaneTemplate spec.template.spec field is immutable. Please create new resource instead." - -func (r *K3sControlPlaneTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error { - return ctrl.NewWebhookManagedBy(mgr). - For(r). - Complete() -} - -// +kubebuilder:webhook:verbs=create;update,path=/mutate-controlplane-cluster-x-k8s-io-v1beta1-k3scontrolplanetemplate,mutating=true,failurePolicy=fail,groups=controlplane.cluster.x-k8s.io,resources=k3scontrolplanetemplates,versions=v1beta1,name=default.k3scontrolplanetemplate.controlplane.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 - -var _ webhook.Defaulter = &K3sControlPlaneTemplate{} - -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (r *K3sControlPlaneTemplate) Default() { - infrabootstrapv1.DefaultK3sConfigSpec(&r.Spec.Template.Spec.K3sConfigSpec) - - r.Spec.Template.Spec.RolloutStrategy = defaultRolloutStrategy(r.Spec.Template.Spec.RolloutStrategy) -} - -// +kubebuilder:webhook:verbs=create;update,path=/validate-controlplane-cluster-x-k8s-io-v1beta1-k3scontrolplanetemplate,mutating=false,failurePolicy=fail,groups=controlplane.cluster.x-k8s.io,resources=k3scontrolplanetemplates,versions=v1beta1,name=validation.k3scontrolplanetemplate.controlplane.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 - -var _ webhook.Validator = &K3sControlPlaneTemplate{} - -// ValidateCreate implements webhook.Validator so a webhook will be registered for the type -func (r *K3sControlPlaneTemplate) ValidateCreate() error { - // NOTE: K3sControlPlaneTemplate is behind ClusterTopology feature gate flag; the web hook - // must prevent creating new objects in case the feature flag is disabled. - if !feature.Gates.Enabled(feature.ClusterTopology) { - return field.Forbidden( - field.NewPath("spec"), - "can be set only if the ClusterTopology feature flag is enabled", - ) - } - - spec := r.Spec.Template.Spec - allErrs := validateK3sControlPlaneTemplateResourceSpec(spec, field.NewPath("spec", "template", "spec")) - allErrs = append(allErrs, validateServerConfiguration(spec.K3sConfigSpec.ServerConfiguration, nil, field.NewPath("spec", "template", "spec", "k3sConfigSpec", "serverConfiguration"))...) - allErrs = append(allErrs, spec.K3sConfigSpec.Validate(field.NewPath("spec", "template", "spec", "k3sConfigSpec"))...) - if len(allErrs) > 0 { - return apierrors.NewInvalid(GroupVersion.WithKind("K3sControlPlaneTemplate").GroupKind(), r.Name, allErrs) - } - return nil -} - -// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (r *K3sControlPlaneTemplate) ValidateUpdate(oldRaw runtime.Object) error { - var allErrs field.ErrorList - old, ok := oldRaw.(*K3sControlPlaneTemplate) - if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a K3sControlPlaneTemplate but got a %T", oldRaw)) - } - - if !reflect.DeepEqual(r.Spec.Template.Spec, old.Spec.Template.Spec) { - allErrs = append(allErrs, - field.Invalid(field.NewPath("spec", "template", "spec"), r, k3sControlPlaneTemplateImmutableMsg), - ) - } - - if len(allErrs) == 0 { - return nil - } - return apierrors.NewInvalid(GroupVersion.WithKind("K3sControlPlaneTemplate").GroupKind(), r.Name, allErrs) -} - -// ValidateDelete implements webhook.Validator so a webhook will be registered for the type -func (r *K3sControlPlaneTemplate) ValidateDelete() error { - return nil -} - -// validateK3sControlPlaneTemplateResourceSpec is a copy of validateK3sControlPlaneSpec which -// only validates the fields in K3sControlPlaneTemplateResourceSpec we care about. -func validateK3sControlPlaneTemplateResourceSpec(s K3sControlPlaneTemplateResourceSpec, pathPrefix *field.Path) field.ErrorList { - return validateRolloutStrategy(s.RolloutStrategy, nil, pathPrefix.Child("rolloutStrategy")) -} diff --git a/controlplane/k3s/api/v1beta1/zz_generated.deepcopy.go b/controlplane/k3s/api/v1beta1/zz_generated.deepcopy.go deleted file mode 100644 index 4364ce5b..00000000 --- a/controlplane/k3s/api/v1beta1/zz_generated.deepcopy.go +++ /dev/null @@ -1,363 +0,0 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by controller-gen. DO NOT EDIT. - -package v1beta1 - -import ( - "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/intstr" - apiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sControlPlane) DeepCopyInto(out *K3sControlPlane) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sControlPlane. -func (in *K3sControlPlane) DeepCopy() *K3sControlPlane { - if in == nil { - return nil - } - out := new(K3sControlPlane) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *K3sControlPlane) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sControlPlaneList) DeepCopyInto(out *K3sControlPlaneList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]K3sControlPlane, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sControlPlaneList. -func (in *K3sControlPlaneList) DeepCopy() *K3sControlPlaneList { - if in == nil { - return nil - } - out := new(K3sControlPlaneList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *K3sControlPlaneList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sControlPlaneMachineTemplate) DeepCopyInto(out *K3sControlPlaneMachineTemplate) { - *out = *in - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.InfrastructureRef = in.InfrastructureRef - if in.NodeDrainTimeout != nil { - in, out := &in.NodeDrainTimeout, &out.NodeDrainTimeout - *out = new(v1.Duration) - **out = **in - } - if in.NodeDeletionTimeout != nil { - in, out := &in.NodeDeletionTimeout, &out.NodeDeletionTimeout - *out = new(v1.Duration) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sControlPlaneMachineTemplate. -func (in *K3sControlPlaneMachineTemplate) DeepCopy() *K3sControlPlaneMachineTemplate { - if in == nil { - return nil - } - out := new(K3sControlPlaneMachineTemplate) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sControlPlaneSpec) DeepCopyInto(out *K3sControlPlaneSpec) { - *out = *in - if in.Replicas != nil { - in, out := &in.Replicas, &out.Replicas - *out = new(int32) - **out = **in - } - in.MachineTemplate.DeepCopyInto(&out.MachineTemplate) - in.K3sConfigSpec.DeepCopyInto(&out.K3sConfigSpec) - if in.RolloutAfter != nil { - in, out := &in.RolloutAfter, &out.RolloutAfter - *out = (*in).DeepCopy() - } - if in.RolloutStrategy != nil { - in, out := &in.RolloutStrategy, &out.RolloutStrategy - *out = new(RolloutStrategy) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sControlPlaneSpec. -func (in *K3sControlPlaneSpec) DeepCopy() *K3sControlPlaneSpec { - if in == nil { - return nil - } - out := new(K3sControlPlaneSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sControlPlaneStatus) DeepCopyInto(out *K3sControlPlaneStatus) { - *out = *in - if in.Version != nil { - in, out := &in.Version, &out.Version - *out = new(string) - **out = **in - } - if in.FailureMessage != nil { - in, out := &in.FailureMessage, &out.FailureMessage - *out = new(string) - **out = **in - } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make(apiv1beta1.Conditions, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sControlPlaneStatus. -func (in *K3sControlPlaneStatus) DeepCopy() *K3sControlPlaneStatus { - if in == nil { - return nil - } - out := new(K3sControlPlaneStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sControlPlaneTemplate) DeepCopyInto(out *K3sControlPlaneTemplate) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sControlPlaneTemplate. -func (in *K3sControlPlaneTemplate) DeepCopy() *K3sControlPlaneTemplate { - if in == nil { - return nil - } - out := new(K3sControlPlaneTemplate) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *K3sControlPlaneTemplate) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sControlPlaneTemplateList) DeepCopyInto(out *K3sControlPlaneTemplateList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]K3sControlPlaneTemplate, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sControlPlaneTemplateList. -func (in *K3sControlPlaneTemplateList) DeepCopy() *K3sControlPlaneTemplateList { - if in == nil { - return nil - } - out := new(K3sControlPlaneTemplateList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *K3sControlPlaneTemplateList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sControlPlaneTemplateMachineTemplate) DeepCopyInto(out *K3sControlPlaneTemplateMachineTemplate) { - *out = *in - if in.NodeDrainTimeout != nil { - in, out := &in.NodeDrainTimeout, &out.NodeDrainTimeout - *out = new(v1.Duration) - **out = **in - } - if in.NodeDeletionTimeout != nil { - in, out := &in.NodeDeletionTimeout, &out.NodeDeletionTimeout - *out = new(v1.Duration) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sControlPlaneTemplateMachineTemplate. -func (in *K3sControlPlaneTemplateMachineTemplate) DeepCopy() *K3sControlPlaneTemplateMachineTemplate { - if in == nil { - return nil - } - out := new(K3sControlPlaneTemplateMachineTemplate) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sControlPlaneTemplateResource) DeepCopyInto(out *K3sControlPlaneTemplateResource) { - *out = *in - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sControlPlaneTemplateResource. -func (in *K3sControlPlaneTemplateResource) DeepCopy() *K3sControlPlaneTemplateResource { - if in == nil { - return nil - } - out := new(K3sControlPlaneTemplateResource) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sControlPlaneTemplateResourceSpec) DeepCopyInto(out *K3sControlPlaneTemplateResourceSpec) { - *out = *in - if in.MachineTemplate != nil { - in, out := &in.MachineTemplate, &out.MachineTemplate - *out = new(K3sControlPlaneTemplateMachineTemplate) - (*in).DeepCopyInto(*out) - } - in.K3sConfigSpec.DeepCopyInto(&out.K3sConfigSpec) - if in.RolloutAfter != nil { - in, out := &in.RolloutAfter, &out.RolloutAfter - *out = (*in).DeepCopy() - } - if in.RolloutStrategy != nil { - in, out := &in.RolloutStrategy, &out.RolloutStrategy - *out = new(RolloutStrategy) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sControlPlaneTemplateResourceSpec. -func (in *K3sControlPlaneTemplateResourceSpec) DeepCopy() *K3sControlPlaneTemplateResourceSpec { - if in == nil { - return nil - } - out := new(K3sControlPlaneTemplateResourceSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *K3sControlPlaneTemplateSpec) DeepCopyInto(out *K3sControlPlaneTemplateSpec) { - *out = *in - in.Template.DeepCopyInto(&out.Template) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new K3sControlPlaneTemplateSpec. -func (in *K3sControlPlaneTemplateSpec) DeepCopy() *K3sControlPlaneTemplateSpec { - if in == nil { - return nil - } - out := new(K3sControlPlaneTemplateSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RollingUpdate) DeepCopyInto(out *RollingUpdate) { - *out = *in - if in.MaxSurge != nil { - in, out := &in.MaxSurge, &out.MaxSurge - *out = new(intstr.IntOrString) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpdate. -func (in *RollingUpdate) DeepCopy() *RollingUpdate { - if in == nil { - return nil - } - out := new(RollingUpdate) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RolloutStrategy) DeepCopyInto(out *RolloutStrategy) { - *out = *in - if in.RollingUpdate != nil { - in, out := &in.RollingUpdate, &out.RollingUpdate - *out = new(RollingUpdate) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutStrategy. -func (in *RolloutStrategy) DeepCopy() *RolloutStrategy { - if in == nil { - return nil - } - out := new(RolloutStrategy) - in.DeepCopyInto(out) - return out -} diff --git a/controlplane/k3s/config/certmanager/certificate.yaml b/controlplane/k3s/config/certmanager/certificate.yaml deleted file mode 100644 index 0f645290..00000000 --- a/controlplane/k3s/config/certmanager/certificate.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for breaking changes -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: selfsigned-issuer - namespace: system -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml - namespace: system -spec: - # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize - dnsNames: - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: $(SERVICE_NAME)-cert # this secret will not be prefixed, since it's not managed by kustomize \ No newline at end of file diff --git a/controlplane/k3s/config/certmanager/kustomization.yaml b/controlplane/k3s/config/certmanager/kustomization.yaml deleted file mode 100644 index bebea5a5..00000000 --- a/controlplane/k3s/config/certmanager/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -resources: -- certificate.yaml - -configurations: -- kustomizeconfig.yaml diff --git a/controlplane/k3s/config/certmanager/kustomizeconfig.yaml b/controlplane/k3s/config/certmanager/kustomizeconfig.yaml deleted file mode 100644 index 28a895a4..00000000 --- a/controlplane/k3s/config/certmanager/kustomizeconfig.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# This configuration is for teaching kustomize how to update name ref and var substitution -nameReference: -- kind: Issuer - group: cert-manager.io - fieldSpecs: - - kind: Certificate - group: cert-manager.io - path: spec/issuerRef/name - -varReference: -- kind: Certificate - group: cert-manager.io - path: spec/commonName -- kind: Certificate - group: cert-manager.io - path: spec/dnsNames -- kind: Certificate - group: cert-manager.io - path: spec/secretName diff --git a/controlplane/k3s/config/crd/bases/controlplane.cluster.x-k8s.io_k3scontrolplanes.yaml b/controlplane/k3s/config/crd/bases/controlplane.cluster.x-k8s.io_k3scontrolplanes.yaml deleted file mode 100644 index d75457a2..00000000 --- a/controlplane/k3s/config/crd/bases/controlplane.cluster.x-k8s.io_k3scontrolplanes.yaml +++ /dev/null @@ -1,710 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.1 - creationTimestamp: null - name: k3scontrolplanes.controlplane.cluster.x-k8s.io -spec: - group: controlplane.cluster.x-k8s.io - names: - categories: - - cluster-api - kind: K3sControlPlane - listKind: K3sControlPlaneList - plural: k3scontrolplanes - shortNames: - - k3scp - singular: k3scontrolplane - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Cluster - jsonPath: .metadata.labels['cluster\.x-k8s\.io/cluster-name'] - name: Cluster - type: string - - description: This denotes whether or not the control plane has the uploaded - k3s-config configmap - jsonPath: .status.initialized - name: Initialized - type: boolean - - description: K3sControlPlane API Server is ready to receive requests - jsonPath: .status.ready - name: API Server Available - type: boolean - - description: Total number of machines desired by this control plane - jsonPath: .spec.replicas - name: Desired - priority: 10 - type: integer - - description: Total number of non-terminated machines targeted by this control - plane - jsonPath: .status.replicas - name: Replicas - type: integer - - description: Total number of fully running and ready control plane machines - jsonPath: .status.readyReplicas - name: Ready - type: integer - - description: Total number of non-terminated machines targeted by this control - plane that have the desired template spec - jsonPath: .status.updatedReplicas - name: Updated - type: integer - - description: Total number of unavailable machines targeted by this control plane - jsonPath: .status.unavailableReplicas - name: Unavailable - type: integer - - description: Time duration since creation of KubeadmControlPlane - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - description: Kubernetes version associated with this control plane - jsonPath: .spec.version - name: Version - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: K3sControlPlane is the Schema for the k3scontrolplanes API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: K3sControlPlaneSpec defines the desired state of K3sControlPlane - properties: - k3sConfigSpec: - description: K3sConfigSpec is a K3sConfigSpec to use for initializing - and joining machines to the control plane. - properties: - agentConfiguration: - description: AgentConfiguration defines the k3s agent configuration. - properties: - kubernetesAgentProcesses: - description: KubernetesAgentProcesses defines the k3s agent - kubernetes processes configuration. - properties: - kubeProxyArgs: - description: KubeProxyArgs Customized flag for kube-proxy - process - items: - type: string - type: array - kubeletArgs: - description: KubeletArgs Customized flag for kubelet process - items: - type: string - type: array - type: object - networking: - description: Networking defines the k3s agent networking configuration. - properties: - nodeExternalIP: - description: NodeExternalIP External IP address to advertise - for node. - type: string - nodeIP: - description: NodeIP IP address to advertise for node. - type: string - resolvConf: - description: ResolvConf Path to Kubelet resolv.conf file. - type: string - type: object - node: - description: Node defines the k3s agent node configuration. - properties: - dataDir: - description: DataDir Folder to hold state. - type: string - lbServerPort: - description: 'LBServerPort Local port for supervisor client - load-balancer. If the supervisor and apiserver are not - colocated an additional port 1 less than this port will - also be used for the apiserver client load-balancer. - (default: 6444)' - type: integer - nodeLabels: - description: NodeLabels registering and starting kubelet - with set of labels. - items: - type: string - type: array - nodeName: - description: NodeName k3s node name. - type: string - nodeTaints: - description: NodeTaints registering and starting kubelet - with set of taints. - items: - type: string - type: array - seLinux: - description: SeLinux Enable SELinux in containerd - type: boolean - type: object - runtime: - description: Runtime defines the k3s agent runtime configuration. - properties: - containerRuntimeEndpoint: - description: ContainerRuntimeEndpoint Disable embedded - containerd and use alternative CRI implementation. - type: string - pauseImage: - description: PauseImage Customized pause image for containerd - or Docker sandbox. - type: string - privateRegistry: - description: PrivateRegistry Path to a private registry - configuration file. - type: string - type: object - type: object - cluster: - description: Cluster defines the k3s cluster Options. - properties: - server: - description: Server which server to connect to, used to join - a cluster. - type: string - token: - description: Token shared secret used to join a server or - agent to a cluster. - type: string - tokenFile: - description: TokenFile file containing the cluster-secret/token. - type: string - type: object - files: - description: Files specifies extra files to be passed to user_data - upon creation. - items: - description: File defines the input for generating write_files - in cloud-init. - properties: - append: - description: Append specifies whether to append Content - to existing file if Path exists. - type: boolean - content: - description: Content is the actual content of the file. - type: string - contentFrom: - description: ContentFrom is a referenced source of content - to populate the file. - properties: - secret: - description: Secret represents a secret that should - populate this file. - properties: - key: - description: Key is the key in the secret's data - map for this value. - type: string - name: - description: Name of the secret in the KubeadmBootstrapConfig's - namespace to use. - type: string - required: - - key - - name - type: object - required: - - secret - type: object - encoding: - description: Encoding specifies the encoding of the file - contents. - enum: - - base64 - - gzip - - gzip+base64 - type: string - owner: - description: Owner specifies the ownership of the file, - e.g. "root:root". - type: string - path: - description: Path specifies the full path on disk where - to store the file. - type: string - permissions: - description: Permissions specifies the permissions to assign - to the file, e.g. "0640". - type: string - required: - - path - type: object - type: array - postK3sCommands: - description: PostK3sCommands specifies extra commands to run after - k3s setup runs - items: - type: string - type: array - preK3sCommands: - description: PreK3sCommands specifies extra commands to run before - k3s setup runs - items: - type: string - type: array - serverConfiguration: - description: ServerConfiguration defines the k3s server configuration. - properties: - agent: - description: Agent is the agent configuration. - properties: - kubernetesAgentProcesses: - description: KubernetesAgentProcesses defines the k3s - agent kubernetes processes configuration. - properties: - kubeProxyArgs: - description: KubeProxyArgs Customized flag for kube-proxy - process - items: - type: string - type: array - kubeletArgs: - description: KubeletArgs Customized flag for kubelet - process - items: - type: string - type: array - type: object - networking: - description: Networking defines the k3s agent networking - configuration. - properties: - nodeExternalIP: - description: NodeExternalIP External IP address to - advertise for node. - type: string - nodeIP: - description: NodeIP IP address to advertise for node. - type: string - resolvConf: - description: ResolvConf Path to Kubelet resolv.conf - file. - type: string - type: object - node: - description: Node defines the k3s agent node configuration. - properties: - dataDir: - description: DataDir Folder to hold state. - type: string - lbServerPort: - description: 'LBServerPort Local port for supervisor - client load-balancer. If the supervisor and apiserver - are not colocated an additional port 1 less than - this port will also be used for the apiserver client - load-balancer. (default: 6444)' - type: integer - nodeLabels: - description: NodeLabels registering and starting kubelet - with set of labels. - items: - type: string - type: array - nodeName: - description: NodeName k3s node name. - type: string - nodeTaints: - description: NodeTaints registering and starting kubelet - with set of taints. - items: - type: string - type: array - seLinux: - description: SeLinux Enable SELinux in containerd - type: boolean - type: object - runtime: - description: Runtime defines the k3s agent runtime configuration. - properties: - containerRuntimeEndpoint: - description: ContainerRuntimeEndpoint Disable embedded - containerd and use alternative CRI implementation. - type: string - pauseImage: - description: PauseImage Customized pause image for - containerd or Docker sandbox. - type: string - privateRegistry: - description: PrivateRegistry Path to a private registry - configuration file. - type: string - type: object - type: object - database: - description: Database is the database configuration. - properties: - clusterInit: - description: ClusterInit initialize a new cluster using - embedded Etcd. - type: boolean - dataStoreCAFile: - description: DataStoreCAFile TLS Certificate Authority - file used to secure datastore backend communication. - type: string - dataStoreCertFile: - description: DataStoreCertFile TLS certification file - used to secure datastore backend communication. - type: string - dataStoreEndPoint: - description: DataStoreEndPoint specify etcd, Mysql, Postgres, - or Sqlite (default) data source name. - type: string - dataStoreKeyFile: - description: DataStoreKeyFile TLS key file used to secure - datastore backend communication. - type: string - type: object - kubernetesComponents: - description: KubernetesComponents is the kubernetes components - configuration. - properties: - disable: - description: 'Disable do not deploy packaged components - and delete any deployed components (valid items: coredns, - servicelb, traefik,local-storage, metrics-server).' - type: string - disableHelmController: - description: DisableHelmController disable Helm controller. - type: boolean - disableKubeProxy: - description: DisableKubeProxy disable running kube-proxy. - type: boolean - disableNetworkPolicy: - description: DisableNetworkPolicy disable k3s default - network policy controller. - type: boolean - type: object - kubernetesProcesses: - description: KubernetesProcesses is the kubernetes processes - configuration. - properties: - kubeAPIServerArg: - description: KubeAPIServerArgs is a customized flag for - kube-apiserver process - items: - type: string - type: array - kubeControllerManagerArgs: - description: KubeControllerManagerArgs is a customized - flag for kube-controller-manager process - items: - type: string - type: array - kubeSchedulerArgs: - description: KubeSchedulerArgs is a customized flag for - kube-scheduler process - items: - type: string - type: array - type: object - listener: - description: Listener is the listener configuration. - properties: - advertiseAddress: - description: AdvertiseAddress IP address that apiserver - uses to advertise to members of the cluster. - type: string - advertisePort: - description: 'AdvertisePort Port that apiserver uses to - advertise to members of the cluster (default: listen-port).' - type: integer - bindAddress: - description: BindAddress k3s bind address. - type: string - httpsListenPort: - description: HTTPSListenPort HTTPS listen port. - type: integer - tlsSan: - description: TLSSan Add additional hostname or IP as a - Subject Alternative Name in the TLS cert. - type: string - type: object - networking: - description: Networking is the networking configuration. - properties: - clusterCIDR: - description: ClusterCIDR Network CIDR to use for pod IPs. - type: string - clusterDNS: - description: ClusterDNS cluster IP for coredns service. - Should be in your service-cidr range. - type: string - clusterDomain: - description: ClusterDomain cluster Domain. - type: string - flannelBackend: - description: 'FlannelBackend One of ‘none’, ‘vxlan’, ‘ipsec’, - ‘host-gw’, or ‘wireguard’. (default: vxlan)' - type: string - serviceCIDR: - description: ServiceCIDR Network CIDR to use for services - IPs. - type: string - serviceNodePortRange: - description: ServiceNodePortRange Port range to reserve - for services with NodePort visibility. - type: string - type: object - type: object - version: - description: Version specifies the k3s version - type: string - type: object - machineTemplate: - description: MachineTemplate contains information about how machines - should be shaped when creating or updating a control plane. - properties: - infrastructureRef: - description: InfrastructureRef is a required reference to a custom - resource offered by an infrastructure provider. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead - of an entire object, this string should contain a valid - JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part - of an object. TODO: this design is not final and this field - is subject to change in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - x-kubernetes-map-type: atomic - metadata: - description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' - properties: - annotations: - additionalProperties: - type: string - description: 'Annotations is an unstructured key value map - stored with a resource that may be set by external tools - to store and retrieve arbitrary metadata. They are not queryable - and should be preserved when modifying objects. More info: - http://kubernetes.io/docs/user-guide/annotations' - type: object - labels: - additionalProperties: - type: string - description: 'Map of string keys and values that can be used - to organize and categorize (scope and select) objects. May - match selectors of replication controllers and services. - More info: http://kubernetes.io/docs/user-guide/labels' - type: object - type: object - nodeDeletionTimeout: - description: NodeDeletionTimeout defines how long the machine - controller will attempt to delete the Node that the Machine - hosts after the Machine is marked for deletion. A duration of - 0 will retry deletion indefinitely. If no value is provided, - the default value for this property of the Machine resource - will be used. - type: string - nodeDrainTimeout: - description: 'NodeDrainTimeout is the total amount of time that - the controller will spend on draining a controlplane node The - default value is 0, meaning that the node can be drained without - any time limitations. NOTE: NodeDrainTimeout is different from - `kubectl drain --timeout`' - type: string - required: - - infrastructureRef - type: object - replicas: - description: Number of desired machines. Defaults to 1. When stacked - etcd is used only odd numbers are permitted, as per [etcd best practice](https://etcd.io/docs/v3.3.12/faq/#why-an-odd-number-of-cluster-members). - This is a pointer to distinguish between explicit zero and not specified. - format: int32 - type: integer - rolloutAfter: - description: RolloutAfter is a field to indicate a rollout should - be performed after the specified time even if no changes have been - made to the K3sControlPlane. - format: date-time - type: string - rolloutStrategy: - default: - rollingUpdate: - maxSurge: 1 - type: RollingUpdate - description: The RolloutStrategy to use to replace control plane machines - with new ones. - properties: - rollingUpdate: - description: Rolling update config params. Present only if RolloutStrategyType - = RollingUpdate. - properties: - maxSurge: - anyOf: - - type: integer - - type: string - description: 'The maximum number of control planes that can - be scheduled above or under the desired number of control - planes. Value can be an absolute number 1 or 0. Defaults - to 1. Example: when this is set to 1, the control plane - can be scaled up immediately when the rolling update starts.' - x-kubernetes-int-or-string: true - type: object - type: - description: Type of rollout. Currently the only supported strategy - is "RollingUpdate". Default is RollingUpdate. - type: string - type: object - version: - description: Version defines the desired K3s version. - type: string - required: - - machineTemplate - - version - type: object - status: - description: K3sControlPlaneStatus defines the observed state of K3sControlPlane - properties: - conditions: - description: Conditions defines current service state of the K3sControlPlane. - items: - description: Condition defines an observation of a Cluster API resource - operational state. - properties: - lastTransitionTime: - description: Last time the condition transitioned from one status - to another. This should be when the underlying condition changed. - If that is not known, then using the time when the API field - changed is acceptable. - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. This field may be empty. - type: string - reason: - description: The reason for the condition's last transition - in CamelCase. The specific API may choose whether or not this - field is considered a guaranteed API. This field may not be - empty. - type: string - severity: - description: Severity provides an explicit classification of - Reason code, so the users or machines can immediately understand - the current situation and act accordingly. The Severity field - MUST be set only when Status=False. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of condition in CamelCase or in foo.example.com/CamelCase. - Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. - type: string - required: - - lastTransitionTime - - status - - type - type: object - type: array - failureMessage: - description: ErrorMessage indicates that there is a terminal problem - reconciling the state, and will be set to a descriptive error message. - type: string - failureReason: - description: FailureReason indicates that there is a terminal problem - reconciling the state, and will be set to a token value suitable - for programmatic interpretation. - type: string - initialized: - description: Initialized denotes whether or not the control plane - has the uploaded kubeadm-config configmap. - type: boolean - observedGeneration: - description: ObservedGeneration is the latest generation observed - by the controller. - format: int64 - type: integer - ready: - description: Ready denotes that the K3sControlPlane API Server is - ready to receive requests. - type: boolean - readyReplicas: - description: Total number of fully running and ready control plane - machines. - format: int32 - type: integer - replicas: - description: Total number of non-terminated machines targeted by this - control plane (their labels match the selector). - format: int32 - type: integer - selector: - description: 'Selector is the label selector in string format to avoid - introspection by clients, and is used to provide the CRD-based integration - for the scale subresource and additional integrations for things - like kubectl describe.. The string will be in the same format as - the query-param syntax. More info about label selectors: http://kubernetes.io/docs/user-guide/labels#label-selectors' - type: string - unavailableReplicas: - description: Total number of unavailable machines targeted by this - control plane. This is the total number of machines that are still - required for the deployment to have 100% available capacity. They - may either be machines that are running but not yet ready or machines - that still have not been created. - format: int32 - type: integer - updatedReplicas: - description: Total number of non-terminated machines targeted by this - control plane that have the desired template spec. - format: int32 - type: integer - version: - description: Version represents the minimum Kubernetes version for - the control plane machines in the cluster. - type: string - type: object - type: object - served: true - storage: true - subresources: - scale: - labelSelectorPath: .status.selector - specReplicasPath: .spec.replicas - statusReplicasPath: .status.replicas - status: {} diff --git a/controlplane/k3s/config/crd/bases/controlplane.cluster.x-k8s.io_k3scontrolplanetemplates.yaml b/controlplane/k3s/config/crd/bases/controlplane.cluster.x-k8s.io_k3scontrolplanetemplates.yaml deleted file mode 100644 index 8164511f..00000000 --- a/controlplane/k3s/config/crd/bases/controlplane.cluster.x-k8s.io_k3scontrolplanetemplates.yaml +++ /dev/null @@ -1,524 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.1 - creationTimestamp: null - name: k3scontrolplanetemplates.controlplane.cluster.x-k8s.io -spec: - group: controlplane.cluster.x-k8s.io - names: - kind: K3sControlPlaneTemplate - listKind: K3sControlPlaneTemplateList - plural: k3scontrolplanetemplates - singular: k3scontrolplanetemplate - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: K3sControlPlaneTemplate is the Schema for the k3scontrolplanetemplates - API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: K3sControlPlaneTemplateSpec defines the desired state of - K3sControlPlaneTemplate - properties: - template: - description: K3sControlPlaneTemplateResource describes the data needed - to create a K3sControlPlane from a template. - properties: - spec: - description: 'K3sControlPlaneTemplateResourceSpec defines the - desired state of K3sControlPlane. NOTE: K3sControlPlaneTemplateResourceSpec - is similar to K3sControlPlaneSpec but omits Replicas and Version - fields. These fields do not make sense on the K3sControlPlaneTemplate, - because they are calculated by the Cluster topology reconciler - during reconciliation and thus cannot be configured on the K3sControlPlaneTemplate.' - properties: - k3sConfigSpec: - description: K3sConfigSpec is a K3sConfigSpec to use for initializing - and joining machines to the control plane. - properties: - agentConfiguration: - description: AgentConfiguration defines the k3s agent - configuration. - properties: - kubernetesAgentProcesses: - description: KubernetesAgentProcesses defines the - k3s agent kubernetes processes configuration. - properties: - kubeProxyArgs: - description: KubeProxyArgs Customized flag for - kube-proxy process - items: - type: string - type: array - kubeletArgs: - description: KubeletArgs Customized flag for kubelet - process - items: - type: string - type: array - type: object - networking: - description: Networking defines the k3s agent networking - configuration. - properties: - nodeExternalIP: - description: NodeExternalIP External IP address - to advertise for node. - type: string - nodeIP: - description: NodeIP IP address to advertise for - node. - type: string - resolvConf: - description: ResolvConf Path to Kubelet resolv.conf - file. - type: string - type: object - node: - description: Node defines the k3s agent node configuration. - properties: - dataDir: - description: DataDir Folder to hold state. - type: string - lbServerPort: - description: 'LBServerPort Local port for supervisor - client load-balancer. If the supervisor and - apiserver are not colocated an additional port - 1 less than this port will also be used for - the apiserver client load-balancer. (default: - 6444)' - type: integer - nodeLabels: - description: NodeLabels registering and starting - kubelet with set of labels. - items: - type: string - type: array - nodeName: - description: NodeName k3s node name. - type: string - nodeTaints: - description: NodeTaints registering and starting - kubelet with set of taints. - items: - type: string - type: array - seLinux: - description: SeLinux Enable SELinux in containerd - type: boolean - type: object - runtime: - description: Runtime defines the k3s agent runtime - configuration. - properties: - containerRuntimeEndpoint: - description: ContainerRuntimeEndpoint Disable - embedded containerd and use alternative CRI - implementation. - type: string - pauseImage: - description: PauseImage Customized pause image - for containerd or Docker sandbox. - type: string - privateRegistry: - description: PrivateRegistry Path to a private - registry configuration file. - type: string - type: object - type: object - cluster: - description: Cluster defines the k3s cluster Options. - properties: - server: - description: Server which server to connect to, used - to join a cluster. - type: string - token: - description: Token shared secret used to join a server - or agent to a cluster. - type: string - tokenFile: - description: TokenFile file containing the cluster-secret/token. - type: string - type: object - files: - description: Files specifies extra files to be passed - to user_data upon creation. - items: - description: File defines the input for generating write_files - in cloud-init. - properties: - append: - description: Append specifies whether to append - Content to existing file if Path exists. - type: boolean - content: - description: Content is the actual content of the - file. - type: string - contentFrom: - description: ContentFrom is a referenced source - of content to populate the file. - properties: - secret: - description: Secret represents a secret that - should populate this file. - properties: - key: - description: Key is the key in the secret's - data map for this value. - type: string - name: - description: Name of the secret in the KubeadmBootstrapConfig's - namespace to use. - type: string - required: - - key - - name - type: object - required: - - secret - type: object - encoding: - description: Encoding specifies the encoding of - the file contents. - enum: - - base64 - - gzip - - gzip+base64 - type: string - owner: - description: Owner specifies the ownership of the - file, e.g. "root:root". - type: string - path: - description: Path specifies the full path on disk - where to store the file. - type: string - permissions: - description: Permissions specifies the permissions - to assign to the file, e.g. "0640". - type: string - required: - - path - type: object - type: array - postK3sCommands: - description: PostK3sCommands specifies extra commands - to run after k3s setup runs - items: - type: string - type: array - preK3sCommands: - description: PreK3sCommands specifies extra commands to - run before k3s setup runs - items: - type: string - type: array - serverConfiguration: - description: ServerConfiguration defines the k3s server - configuration. - properties: - agent: - description: Agent is the agent configuration. - properties: - kubernetesAgentProcesses: - description: KubernetesAgentProcesses defines - the k3s agent kubernetes processes configuration. - properties: - kubeProxyArgs: - description: KubeProxyArgs Customized flag - for kube-proxy process - items: - type: string - type: array - kubeletArgs: - description: KubeletArgs Customized flag for - kubelet process - items: - type: string - type: array - type: object - networking: - description: Networking defines the k3s agent - networking configuration. - properties: - nodeExternalIP: - description: NodeExternalIP External IP address - to advertise for node. - type: string - nodeIP: - description: NodeIP IP address to advertise - for node. - type: string - resolvConf: - description: ResolvConf Path to Kubelet resolv.conf - file. - type: string - type: object - node: - description: Node defines the k3s agent node configuration. - properties: - dataDir: - description: DataDir Folder to hold state. - type: string - lbServerPort: - description: 'LBServerPort Local port for - supervisor client load-balancer. If the - supervisor and apiserver are not colocated - an additional port 1 less than this port - will also be used for the apiserver client - load-balancer. (default: 6444)' - type: integer - nodeLabels: - description: NodeLabels registering and starting - kubelet with set of labels. - items: - type: string - type: array - nodeName: - description: NodeName k3s node name. - type: string - nodeTaints: - description: NodeTaints registering and starting - kubelet with set of taints. - items: - type: string - type: array - seLinux: - description: SeLinux Enable SELinux in containerd - type: boolean - type: object - runtime: - description: Runtime defines the k3s agent runtime - configuration. - properties: - containerRuntimeEndpoint: - description: ContainerRuntimeEndpoint Disable - embedded containerd and use alternative - CRI implementation. - type: string - pauseImage: - description: PauseImage Customized pause image - for containerd or Docker sandbox. - type: string - privateRegistry: - description: PrivateRegistry Path to a private - registry configuration file. - type: string - type: object - type: object - database: - description: Database is the database configuration. - properties: - clusterInit: - description: ClusterInit initialize a new cluster - using embedded Etcd. - type: boolean - dataStoreCAFile: - description: DataStoreCAFile TLS Certificate Authority - file used to secure datastore backend communication. - type: string - dataStoreCertFile: - description: DataStoreCertFile TLS certification - file used to secure datastore backend communication. - type: string - dataStoreEndPoint: - description: DataStoreEndPoint specify etcd, Mysql, - Postgres, or Sqlite (default) data source name. - type: string - dataStoreKeyFile: - description: DataStoreKeyFile TLS key file used - to secure datastore backend communication. - type: string - type: object - kubernetesComponents: - description: KubernetesComponents is the kubernetes - components configuration. - properties: - disable: - description: 'Disable do not deploy packaged components - and delete any deployed components (valid items: - coredns, servicelb, traefik,local-storage, metrics-server).' - type: string - disableHelmController: - description: DisableHelmController disable Helm - controller. - type: boolean - disableKubeProxy: - description: DisableKubeProxy disable running - kube-proxy. - type: boolean - disableNetworkPolicy: - description: DisableNetworkPolicy disable k3s - default network policy controller. - type: boolean - type: object - kubernetesProcesses: - description: KubernetesProcesses is the kubernetes - processes configuration. - properties: - kubeAPIServerArg: - description: KubeAPIServerArgs is a customized - flag for kube-apiserver process - items: - type: string - type: array - kubeControllerManagerArgs: - description: KubeControllerManagerArgs is a customized - flag for kube-controller-manager process - items: - type: string - type: array - kubeSchedulerArgs: - description: KubeSchedulerArgs is a customized - flag for kube-scheduler process - items: - type: string - type: array - type: object - listener: - description: Listener is the listener configuration. - properties: - advertiseAddress: - description: AdvertiseAddress IP address that - apiserver uses to advertise to members of the - cluster. - type: string - advertisePort: - description: 'AdvertisePort Port that apiserver - uses to advertise to members of the cluster - (default: listen-port).' - type: integer - bindAddress: - description: BindAddress k3s bind address. - type: string - httpsListenPort: - description: HTTPSListenPort HTTPS listen port. - type: integer - tlsSan: - description: TLSSan Add additional hostname or - IP as a Subject Alternative Name in the TLS - cert. - type: string - type: object - networking: - description: Networking is the networking configuration. - properties: - clusterCIDR: - description: ClusterCIDR Network CIDR to use for - pod IPs. - type: string - clusterDNS: - description: ClusterDNS cluster IP for coredns - service. Should be in your service-cidr range. - type: string - clusterDomain: - description: ClusterDomain cluster Domain. - type: string - flannelBackend: - description: 'FlannelBackend One of ‘none’, ‘vxlan’, - ‘ipsec’, ‘host-gw’, or ‘wireguard’. (default: - vxlan)' - type: string - serviceCIDR: - description: ServiceCIDR Network CIDR to use for - services IPs. - type: string - serviceNodePortRange: - description: ServiceNodePortRange Port range to - reserve for services with NodePort visibility. - type: string - type: object - type: object - version: - description: Version specifies the k3s version - type: string - type: object - machineTemplate: - description: MachineTemplate contains information about how - machines should be shaped when creating or updating a control - plane. - properties: - nodeDeletionTimeout: - description: NodeDeletionTimeout defines how long the - machine controller will attempt to delete the Node that - the Machine hosts after the Machine is marked for deletion. - A duration of 0 will retry deletion indefinitely. If - no value is provided, the default value for this property - of the Machine resource will be used. - type: string - nodeDrainTimeout: - description: 'NodeDrainTimeout is the total amount of - time that the controller will spend on draining a controlplane - node The default value is 0, meaning that the node can - be drained without any time limitations. NOTE: NodeDrainTimeout - is different from `kubectl drain --timeout`' - type: string - type: object - rolloutAfter: - description: RolloutAfter is a field to indicate a rollout - should be performed after the specified time even if no - changes have been made to the K3sControlPlane. - format: date-time - type: string - rolloutStrategy: - default: - rollingUpdate: - maxSurge: 1 - type: RollingUpdate - description: The RolloutStrategy to use to replace control - plane machines with new ones. - properties: - rollingUpdate: - description: Rolling update config params. Present only - if RolloutStrategyType = RollingUpdate. - properties: - maxSurge: - anyOf: - - type: integer - - type: string - description: 'The maximum number of control planes - that can be scheduled above or under the desired - number of control planes. Value can be an absolute - number 1 or 0. Defaults to 1. Example: when this - is set to 1, the control plane can be scaled up - immediately when the rolling update starts.' - x-kubernetes-int-or-string: true - type: object - type: - description: Type of rollout. Currently the only supported - strategy is "RollingUpdate". Default is RollingUpdate. - type: string - type: object - required: - - k3sConfigSpec - type: object - required: - - spec - type: object - required: - - template - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/controlplane/k3s/config/crd/kustomization.yaml b/controlplane/k3s/config/crd/kustomization.yaml deleted file mode 100644 index 7ca630c2..00000000 --- a/controlplane/k3s/config/crd/kustomization.yaml +++ /dev/null @@ -1,27 +0,0 @@ -commonLabels: - cluster.x-k8s.io/v1beta1: v1beta1 - -# This kustomization.yaml is not intended to be run by itself, -# since it depends on service name and namespace that are out of this kustomize package. -# It should be run by config/default -resources: -- bases/controlplane.cluster.x-k8s.io_k3scontrolplanes.yaml -- bases/controlplane.cluster.x-k8s.io_k3scontrolplanetemplates.yaml -#+kubebuilder:scaffold:crdkustomizeresource - -patchesStrategicMerge: -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. -# patches here are for enabling the conversion webhook for each CRD -- patches/webhook_in_k3scontrolplanes.yaml -- patches/webhook_in_k3scontrolplanetemplates.yaml -#+kubebuilder:scaffold:crdkustomizewebhookpatch - -# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. -# patches here are for enabling the CA injection for each CRD -- patches/cainjection_in_k3scontrolplanes.yaml -- patches/cainjection_in_k3scontrolplanetemplates.yaml -#+kubebuilder:scaffold:crdkustomizecainjectionpatch - -# the following config is for teaching kustomize how to do kustomization for CRDs. -configurations: -- kustomizeconfig.yaml diff --git a/controlplane/k3s/config/crd/kustomizeconfig.yaml b/controlplane/k3s/config/crd/kustomizeconfig.yaml deleted file mode 100644 index ec5c150a..00000000 --- a/controlplane/k3s/config/crd/kustomizeconfig.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# This file is for teaching kustomize how to substitute name and namespace reference in CRD -nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/name - -namespace: -- kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/namespace - create: false - -varReference: -- path: metadata/annotations diff --git a/controlplane/k3s/config/crd/patches/cainjection_in_k3scontrolplanes.yaml b/controlplane/k3s/config/crd/patches/cainjection_in_k3scontrolplanes.yaml deleted file mode 100644 index 8a4b3869..00000000 --- a/controlplane/k3s/config/crd/patches/cainjection_in_k3scontrolplanes.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: k3scontrolplanes.controlplane.cluster.x-k8s.io diff --git a/controlplane/k3s/config/crd/patches/cainjection_in_k3scontrolplanetemplates.yaml b/controlplane/k3s/config/crd/patches/cainjection_in_k3scontrolplanetemplates.yaml deleted file mode 100644 index a5779327..00000000 --- a/controlplane/k3s/config/crd/patches/cainjection_in_k3scontrolplanetemplates.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: k3scontrolplanetemplates.controlplane.cluster.x-k8s.io diff --git a/controlplane/k3s/config/crd/patches/webhook_in_k3scontrolplanes.yaml b/controlplane/k3s/config/crd/patches/webhook_in_k3scontrolplanes.yaml deleted file mode 100644 index a96c014f..00000000 --- a/controlplane/k3s/config/crd/patches/webhook_in_k3scontrolplanes.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: k3scontrolplanes.controlplane.cluster.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: ["v1", "v1beta1"] - clientConfig: - # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, - # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) - caBundle: Cg== - service: - namespace: system - name: webhook-service - path: /convert diff --git a/controlplane/k3s/config/crd/patches/webhook_in_k3scontrolplanetemplates.yaml b/controlplane/k3s/config/crd/patches/webhook_in_k3scontrolplanetemplates.yaml deleted file mode 100644 index 53e5e9c0..00000000 --- a/controlplane/k3s/config/crd/patches/webhook_in_k3scontrolplanetemplates.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: k3scontrolplanetemplates.controlplane.cluster.x-k8s.io -spec: - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: ["v1", "v1beta1"] - clientConfig: - # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, - # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) - caBundle: Cg== - service: - namespace: system - name: webhook-service - path: /convert diff --git a/controlplane/k3s/config/default/kustomization.yaml b/controlplane/k3s/config/default/kustomization.yaml deleted file mode 100644 index 439bb3bd..00000000 --- a/controlplane/k3s/config/default/kustomization.yaml +++ /dev/null @@ -1,58 +0,0 @@ -namespace: capkk-k3s-control-plane-system - -namePrefix: capkk-k3s-control-plane- - -commonLabels: - cluster.x-k8s.io/provider: "control-plane-k3s" - -resources: - - namespace.yaml - -bases: - - ../crd - - ../rbac - - ../manager - - ../webhook - - ../certmanager - -patchesStrategicMerge: - # Provide customizable hook for make targets. - - manager_image_patch.yaml - - manager_pull_policy.yaml - # Enable webhook. - - manager_webhook_patch.yaml - # Inject certificate in the webhook definition. - - webhookcainjection_patch.yaml - # Enable aggregated ClusterRole aggregation - - manager_role_aggregation_patch.yaml - -vars: - - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR - objref: - kind: Certificate - group: cert-manager.io - version: v1 - name: serving-cert # this name should match the one in certificate.yaml - fieldref: - fieldpath: metadata.namespace - - name: CERTIFICATE_NAME - objref: - kind: Certificate - group: cert-manager.io - version: v1 - name: serving-cert # this name should match the one in certificate.yaml - - name: SERVICE_NAMESPACE # namespace of the service - objref: - kind: Service - version: v1 - name: webhook-service - fieldref: - fieldpath: metadata.namespace - - name: SERVICE_NAME - objref: - kind: Service - version: v1 - name: webhook-service - -configurations: - - kustomizeconfig.yaml diff --git a/controlplane/k3s/config/default/kustomizeconfig.yaml b/controlplane/k3s/config/default/kustomizeconfig.yaml deleted file mode 100644 index eb191e64..00000000 --- a/controlplane/k3s/config/default/kustomizeconfig.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# This configuration is for teaching kustomize how to update name ref and var substitution -varReference: -- kind: Deployment - path: spec/template/spec/volumes/secret/secretName diff --git a/controlplane/k3s/config/default/manager_image_patch.yaml b/controlplane/k3s/config/default/manager_image_patch.yaml deleted file mode 100644 index ac1f04b8..00000000 --- a/controlplane/k3s/config/default/manager_image_patch.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - image: docker.io/kubespheredev/k3s-control-plane-controller:main - name: manager diff --git a/controlplane/k3s/config/default/manager_pull_policy.yaml b/controlplane/k3s/config/default/manager_pull_policy.yaml deleted file mode 100644 index 74a0879c..00000000 --- a/controlplane/k3s/config/default/manager_pull_policy.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - imagePullPolicy: Always diff --git a/controlplane/k3s/config/default/manager_role_aggregation_patch.yaml b/controlplane/k3s/config/default/manager_role_aggregation_patch.yaml deleted file mode 100644 index 25399dfe..00000000 --- a/controlplane/k3s/config/default/manager_role_aggregation_patch.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: manager-role - labels: - kubeadm.controlplane.cluster.x-k8s.io/aggregate-to-manager: "true" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: aggregated-manager-role diff --git a/controlplane/k3s/config/default/manager_webhook_patch.yaml b/controlplane/k3s/config/default/manager_webhook_patch.yaml deleted file mode 100644 index bccef6d7..00000000 --- a/controlplane/k3s/config/default/manager_webhook_patch.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - ports: - - containerPort: 9443 - name: webhook-server - protocol: TCP - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - volumes: - - name: cert - secret: - secretName: $(SERVICE_NAME)-cert diff --git a/controlplane/k3s/config/default/namespace.yaml b/controlplane/k3s/config/default/namespace.yaml deleted file mode 100644 index 8b55c3cd..00000000 --- a/controlplane/k3s/config/default/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: controller-manager - name: system diff --git a/controlplane/k3s/config/default/webhookcainjection_patch.yaml b/controlplane/k3s/config/default/webhookcainjection_patch.yaml deleted file mode 100644 index 02ab515d..00000000 --- a/controlplane/k3s/config/default/webhookcainjection_patch.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# This patch add annotation to admission webhook config and -# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: mutating-webhook-configuration - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validating-webhook-configuration - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/controlplane/k3s/config/manager/kustomization.yaml b/controlplane/k3s/config/manager/kustomization.yaml deleted file mode 100644 index 7394a6d0..00000000 --- a/controlplane/k3s/config/manager/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: - - manager.yaml diff --git a/controlplane/k3s/config/manager/manager.yaml b/controlplane/k3s/config/manager/manager.yaml deleted file mode 100644 index 380e8564..00000000 --- a/controlplane/k3s/config/manager/manager.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system - labels: - control-plane: controller-manager -spec: - selector: - matchLabels: - control-plane: controller-manager - replicas: 1 - template: - metadata: - labels: - control-plane: controller-manager - spec: - containers: - - command: - - /manager - args: - - "--leader-elect" - - "--metrics-bind-addr=localhost:8080" - - "--v=5" - image: controller:latest - name: manager - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - ports: - - containerPort: 9440 - name: healthz - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: healthz - livenessProbe: - httpGet: - path: /healthz - port: healthz - terminationGracePeriodSeconds: 10 - serviceAccountName: manager - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane diff --git a/controlplane/k3s/config/rbac/aggregated_role.yaml b/controlplane/k3s/config/rbac/aggregated_role.yaml deleted file mode 100644 index 7c07f5a5..00000000 --- a/controlplane/k3s/config/rbac/aggregated_role.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: aggregated-manager-role -aggregationRule: - clusterRoleSelectors: - - matchLabels: - kubeadm.controlplane.cluster.x-k8s.io/aggregate-to-manager: "true" -rules: [] diff --git a/controlplane/k3s/config/rbac/kustomization.yaml b/controlplane/k3s/config/rbac/kustomization.yaml deleted file mode 100644 index 03140358..00000000 --- a/controlplane/k3s/config/rbac/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -resources: -- service_account.yaml -- role.yaml -- role_binding.yaml -- leader_election_role.yaml -- leader_election_role_binding.yaml -- aggregated_role.yaml diff --git a/controlplane/k3s/config/rbac/leader_election_role.yaml b/controlplane/k3s/config/rbac/leader_election_role.yaml deleted file mode 100644 index 4190ec80..00000000 --- a/controlplane/k3s/config/rbac/leader_election_role.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# permissions to do leader election. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: leader-election-role -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch diff --git a/controlplane/k3s/config/rbac/leader_election_role_binding.yaml b/controlplane/k3s/config/rbac/leader_election_role_binding.yaml deleted file mode 100644 index a73dfa95..00000000 --- a/controlplane/k3s/config/rbac/leader_election_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: leader-election-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: leader-election-role -subjects: - - kind: ServiceAccount - name: manager - namespace: system diff --git a/controlplane/k3s/config/rbac/role.yaml b/controlplane/k3s/config/rbac/role.yaml deleted file mode 100644 index edf8324f..00000000 --- a/controlplane/k3s/config/rbac/role.yaml +++ /dev/null @@ -1,72 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: manager-role -rules: -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - get - - list - - watch -- apiGroups: - - bootstrap.cluster.x-k8s.io - - controlplane.cluster.x-k8s.io - - infrastructure.cluster.x-k8s.io - resources: - - '*' - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - cluster.x-k8s.io - resources: - - clusters - - clusters/status - verbs: - - get - - list - - watch -- apiGroups: - - cluster.x-k8s.io - resources: - - machines - - machines/status - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - get - - list - - patch - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - create - - get - - list - - patch - - update - - watch diff --git a/controlplane/k3s/config/rbac/role_binding.yaml b/controlplane/k3s/config/rbac/role_binding.yaml deleted file mode 100644 index 3ffc9c2e..00000000 --- a/controlplane/k3s/config/rbac/role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: manager-role -subjects: - - kind: ServiceAccount - name: manager - namespace: system diff --git a/controlplane/k3s/config/rbac/service_account.yaml b/controlplane/k3s/config/rbac/service_account.yaml deleted file mode 100644 index 77f747b5..00000000 --- a/controlplane/k3s/config/rbac/service_account.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: manager - namespace: system diff --git a/controlplane/k3s/config/samples/controlplane_v1beta1_k3scontrolplane.yaml b/controlplane/k3s/config/samples/controlplane_v1beta1_k3scontrolplane.yaml deleted file mode 100644 index b89a0562..00000000 --- a/controlplane/k3s/config/samples/controlplane_v1beta1_k3scontrolplane.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: K3sControlPlane -metadata: - name: k3scontrolplane-sample -spec: - # TODO(user): Add fields here diff --git a/controlplane/k3s/config/samples/controlplane_v1beta1_k3scontrolplanetemplate.yaml b/controlplane/k3s/config/samples/controlplane_v1beta1_k3scontrolplanetemplate.yaml deleted file mode 100644 index 3c8aa7d9..00000000 --- a/controlplane/k3s/config/samples/controlplane_v1beta1_k3scontrolplanetemplate.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: K3sControlPlaneTemplate -metadata: - name: k3scontrolplanetemplate-sample -spec: - # TODO(user): Add fields here diff --git a/controlplane/k3s/config/samples/kustomization.yaml b/controlplane/k3s/config/samples/kustomization.yaml deleted file mode 100644 index bd527764..00000000 --- a/controlplane/k3s/config/samples/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -## Append samples you want in your CSV to this file as resources ## -resources: -- controlplane_v1beta1_k3scontrolplane.yaml -- controlplane_v1beta1_k3scontrolplanetemplate.yaml -#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/controlplane/k3s/config/scorecard/bases/config.yaml b/controlplane/k3s/config/scorecard/bases/config.yaml deleted file mode 100644 index c7704784..00000000 --- a/controlplane/k3s/config/scorecard/bases/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: scorecard.operatorframework.io/v1alpha3 -kind: Configuration -metadata: - name: config -stages: -- parallel: true - tests: [] diff --git a/controlplane/k3s/config/scorecard/kustomization.yaml b/controlplane/k3s/config/scorecard/kustomization.yaml deleted file mode 100644 index 50cd2d08..00000000 --- a/controlplane/k3s/config/scorecard/kustomization.yaml +++ /dev/null @@ -1,16 +0,0 @@ -resources: -- bases/config.yaml -patchesJson6902: -- path: patches/basic.config.yaml - target: - group: scorecard.operatorframework.io - version: v1alpha3 - kind: Configuration - name: config -- path: patches/olm.config.yaml - target: - group: scorecard.operatorframework.io - version: v1alpha3 - kind: Configuration - name: config -#+kubebuilder:scaffold:patchesJson6902 diff --git a/controlplane/k3s/config/scorecard/patches/basic.config.yaml b/controlplane/k3s/config/scorecard/patches/basic.config.yaml deleted file mode 100644 index 90f7ef77..00000000 --- a/controlplane/k3s/config/scorecard/patches/basic.config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - basic-check-spec - image: quay.io/operator-framework/scorecard-test:v1.23.0 - labels: - suite: basic - test: basic-check-spec-test diff --git a/controlplane/k3s/config/scorecard/patches/olm.config.yaml b/controlplane/k3s/config/scorecard/patches/olm.config.yaml deleted file mode 100644 index b55840e1..00000000 --- a/controlplane/k3s/config/scorecard/patches/olm.config.yaml +++ /dev/null @@ -1,50 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-bundle-validation - image: quay.io/operator-framework/scorecard-test:v1.23.0 - labels: - suite: olm - test: olm-bundle-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-validation - image: quay.io/operator-framework/scorecard-test:v1.23.0 - labels: - suite: olm - test: olm-crds-have-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-resources - image: quay.io/operator-framework/scorecard-test:v1.23.0 - labels: - suite: olm - test: olm-crds-have-resources-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-spec-descriptors - image: quay.io/operator-framework/scorecard-test:v1.23.0 - labels: - suite: olm - test: olm-spec-descriptors-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-status-descriptors - image: quay.io/operator-framework/scorecard-test:v1.23.0 - labels: - suite: olm - test: olm-status-descriptors-test diff --git a/controlplane/k3s/config/webhook/kustomization.yaml b/controlplane/k3s/config/webhook/kustomization.yaml deleted file mode 100644 index 9cf26134..00000000 --- a/controlplane/k3s/config/webhook/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -resources: -- manifests.yaml -- service.yaml - -configurations: -- kustomizeconfig.yaml diff --git a/controlplane/k3s/config/webhook/kustomizeconfig.yaml b/controlplane/k3s/config/webhook/kustomizeconfig.yaml deleted file mode 100644 index 25e21e3c..00000000 --- a/controlplane/k3s/config/webhook/kustomizeconfig.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# the following config is for teaching kustomize where to look at when substituting vars. -# It requires kustomize v2.1.0 or newer to work properly. -nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: MutatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/name - - kind: ValidatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/name - -namespace: -- kind: MutatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/namespace - create: true -- kind: ValidatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/namespace - create: true - -varReference: -- path: metadata/annotations diff --git a/controlplane/k3s/config/webhook/manifests.yaml b/controlplane/k3s/config/webhook/manifests.yaml deleted file mode 100644 index a1d142c8..00000000 --- a/controlplane/k3s/config/webhook/manifests.yaml +++ /dev/null @@ -1,100 +0,0 @@ ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - creationTimestamp: null - name: mutating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-controlplane-cluster-x-k8s-io-v1beta1-k3scontrolplane - failurePolicy: Fail - matchPolicy: Equivalent - name: default.k3scontrolplane.controlplane.cluster.x-k8s.io - rules: - - apiGroups: - - controlplane.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - k3scontrolplanes - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-controlplane-cluster-x-k8s-io-v1beta1-k3scontrolplanetemplate - failurePolicy: Fail - name: default.k3scontrolplanetemplate.controlplane.cluster.x-k8s.io - rules: - - apiGroups: - - controlplane.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - k3scontrolplanetemplates - sideEffects: None ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - creationTimestamp: null - name: validating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /validate-controlplane-cluster-x-k8s-io-v1beta1-k3scontrolplane - failurePolicy: Fail - matchPolicy: Equivalent - name: validation.k3scontrolplane.controlplane.cluster.x-k8s.io - rules: - - apiGroups: - - controlplane.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - k3scontrolplanes - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /validate-controlplane-cluster-x-k8s-io-v1beta1-k3scontrolplanetemplate - failurePolicy: Fail - name: validation.k3scontrolplanetemplate.controlplane.cluster.x-k8s.io - rules: - - apiGroups: - - controlplane.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - k3scontrolplanetemplates - sideEffects: None diff --git a/controlplane/k3s/config/webhook/service.yaml b/controlplane/k3s/config/webhook/service.yaml deleted file mode 100644 index 9bc95014..00000000 --- a/controlplane/k3s/config/webhook/service.yaml +++ /dev/null @@ -1,10 +0,0 @@ - -apiVersion: v1 -kind: Service -metadata: - name: webhook-service - namespace: system -spec: - ports: - - port: 443 - targetPort: webhook-server diff --git a/controlplane/k3s/controllers/consts.go b/controlplane/k3s/controllers/consts.go deleted file mode 100644 index c5545ffc..00000000 --- a/controlplane/k3s/controllers/consts.go +++ /dev/null @@ -1,35 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package controllers - -import ( - "time" -) - -const ( - // deleteRequeueAfter is how long to wait before checking again to see if - // all control plane machines have been deleted. - deleteRequeueAfter = 30 * time.Second - - // preflightFailedRequeueAfter is how long to wait before trying to scale - // up/down if some preflight check for those operation has failed. - preflightFailedRequeueAfter = 15 * time.Second - - // dependentCertRequeueAfter is how long to wait before checking again to see if - // dependent certificates have been created. - dependentCertRequeueAfter = 30 * time.Second -) diff --git a/controlplane/k3s/controllers/doc.go b/controlplane/k3s/controllers/doc.go deleted file mode 100644 index e2710628..00000000 --- a/controlplane/k3s/controllers/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package controllers contains k3s control plane controller. -package controllers diff --git a/controlplane/k3s/controllers/helpers.go b/controlplane/k3s/controllers/helpers.go deleted file mode 100644 index 3a5b924f..00000000 --- a/controlplane/k3s/controllers/helpers.go +++ /dev/null @@ -1,301 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package controllers - -import ( - "context" - "encoding/json" - "strings" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - kerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apiserver/pkg/storage/names" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/external" - controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" - "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/cluster-api/util/conditions" - utilconversion "sigs.k8s.io/cluster-api/util/conversion" - "sigs.k8s.io/cluster-api/util/patch" - ctrl "sigs.k8s.io/controller-runtime" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" - k3sCluster "github.com/kubesphere/kubekey/v3/controlplane/k3s/pkg/cluster" - "github.com/kubesphere/kubekey/v3/controlplane/k3s/pkg/kubeconfig" - "github.com/kubesphere/kubekey/v3/util/secret" -) - -func (r *K3sControlPlaneReconciler) reconcileKubeconfig(ctx context.Context, cluster *clusterv1.Cluster, kcp *infracontrolplanev1.K3sControlPlane) (ctrl.Result, error) { - endpoint := cluster.Spec.ControlPlaneEndpoint - if endpoint.IsZero() { - return ctrl.Result{}, nil - } - - controllerOwnerRef := *metav1.NewControllerRef(kcp, infracontrolplanev1.GroupVersion.WithKind("K3sControlPlane")) - clusterName := util.ObjectKey(cluster) - configSecret, err := secret.GetFromNamespacedName(ctx, r.Client, clusterName, secret.Kubeconfig) - switch { - case apierrors.IsNotFound(err): - createErr := kubeconfig.CreateSecretWithOwner( - ctx, - r.Client, - clusterName, - endpoint.String(), - controllerOwnerRef, - ) - if errors.Is(createErr, kubeconfig.ErrDependentCertificateNotFound) { - return ctrl.Result{RequeueAfter: dependentCertRequeueAfter}, nil - } - // always return if we have just created in order to skip rotation checks - return ctrl.Result{}, createErr - case err != nil: - return ctrl.Result{}, errors.Wrap(err, "failed to retrieve kubeconfig Secret") - } - - // check if the kubeconfig secret was created by controllers, and thus it has the Cluster as the owner instead of KCP; - // if yes, adopt it. - if util.IsOwnedByObject(configSecret, cluster) && !util.IsControlledBy(configSecret, kcp) { - if err := r.adoptKubeconfigSecret(ctx, cluster, configSecret, controllerOwnerRef); err != nil { - return ctrl.Result{}, err - } - } - - // only do rotation on owned secrets - if !util.IsControlledBy(configSecret, kcp) { - return ctrl.Result{}, nil - } - - return ctrl.Result{}, nil -} - -func (r *K3sControlPlaneReconciler) adoptKubeconfigSecret(ctx context.Context, cluster *clusterv1.Cluster, configSecret *corev1.Secret, controllerOwnerRef metav1.OwnerReference) error { - log := ctrl.LoggerFrom(ctx) - log.Info("Adopting KubeConfig secret created by controllers", "Name", configSecret.Name) - - patch, err := patch.NewHelper(configSecret, r.Client) - if err != nil { - return errors.Wrap(err, "failed to create patch helper for the kubeconfig secret") - } - configSecret.OwnerReferences = util.RemoveOwnerRef(configSecret.OwnerReferences, metav1.OwnerReference{ - APIVersion: clusterv1.GroupVersion.String(), - Kind: "Cluster", - Name: cluster.Name, - UID: cluster.UID, - }) - configSecret.OwnerReferences = util.EnsureOwnerRef(configSecret.OwnerReferences, controllerOwnerRef) - if err := patch.Patch(ctx, configSecret); err != nil { - return errors.Wrap(err, "failed to patch the kubeconfig secret") - } - return nil -} - -func (r *K3sControlPlaneReconciler) reconcileExternalReference(ctx context.Context, cluster *clusterv1.Cluster, ref *corev1.ObjectReference) error { - if !strings.HasSuffix(ref.Kind, clusterv1.TemplateSuffix) { - return nil - } - - if err := utilconversion.UpdateReferenceAPIContract(ctx, r.Client, r.APIReader, ref); err != nil { - return err - } - - obj, err := external.Get(ctx, r.Client, ref, cluster.Namespace) - if err != nil { - return err - } - - // Note: We intentionally do not handle checking for the paused label on an external template reference - - patchHelper, err := patch.NewHelper(obj, r.Client) - if err != nil { - return err - } - - obj.SetOwnerReferences(util.EnsureOwnerRef(obj.GetOwnerReferences(), metav1.OwnerReference{ - APIVersion: clusterv1.GroupVersion.String(), - Kind: "Cluster", - Name: cluster.Name, - UID: cluster.UID, - })) - - return patchHelper.Patch(ctx, obj) -} - -func (r *K3sControlPlaneReconciler) cloneConfigsAndGenerateMachine(ctx context.Context, cluster *clusterv1.Cluster, kcp *infracontrolplanev1.K3sControlPlane, bootstrapSpec *infrabootstrapv1.K3sConfigSpec, failureDomain *string) error { - var errs []error - - // Since the cloned resource should eventually have a controller ref for the Machine, we create an - // OwnerReference here without the Controller field set - infraCloneOwner := &metav1.OwnerReference{ - APIVersion: infracontrolplanev1.GroupVersion.String(), - Kind: "K3sControlPlane", - Name: kcp.Name, - UID: kcp.UID, - } - - // Clone the infrastructure template - infraRef, err := external.CloneTemplate(ctx, &external.CloneTemplateInput{ - Client: r.Client, - TemplateRef: &kcp.Spec.MachineTemplate.InfrastructureRef, - Namespace: kcp.Namespace, - OwnerRef: infraCloneOwner, - ClusterName: cluster.Name, - Labels: k3sCluster.ControlPlaneMachineLabelsForCluster(kcp, cluster.Name), - Annotations: kcp.Spec.MachineTemplate.ObjectMeta.Annotations, - }) - if err != nil { - // Safe to return early here since no resources have been created yet. - conditions.MarkFalse(kcp, infracontrolplanev1.MachinesCreatedCondition, infracontrolplanev1.InfrastructureTemplateCloningFailedReason, - clusterv1.ConditionSeverityError, err.Error()) - return errors.Wrap(err, "failed to clone infrastructure template") - } - - // Clone the bootstrap configuration - bootstrapRef, err := r.generateK3sConfig(ctx, kcp, cluster, bootstrapSpec) - if err != nil { - conditions.MarkFalse(kcp, infracontrolplanev1.MachinesCreatedCondition, infracontrolplanev1.BootstrapTemplateCloningFailedReason, - clusterv1.ConditionSeverityError, err.Error()) - errs = append(errs, errors.Wrap(err, "failed to generate bootstrap config")) - } - - // Only proceed to generating the Machine if we haven't encountered an error - if len(errs) == 0 { - if err := r.generateMachine(ctx, kcp, cluster, infraRef, bootstrapRef, failureDomain); err != nil { - conditions.MarkFalse(kcp, infracontrolplanev1.MachinesCreatedCondition, infracontrolplanev1.MachineGenerationFailedReason, - clusterv1.ConditionSeverityError, err.Error()) - errs = append(errs, errors.Wrap(err, "failed to create Machine")) - } - } - - // If we encountered any errors, attempt to clean up any dangling resources - if len(errs) > 0 { - if err := r.cleanupFromGeneration(ctx, infraRef, bootstrapRef); err != nil { - errs = append(errs, errors.Wrap(err, "failed to cleanup generated resources")) - } - - return kerrors.NewAggregate(errs) - } - - return nil -} - -func (r *K3sControlPlaneReconciler) cleanupFromGeneration(ctx context.Context, remoteRefs ...*corev1.ObjectReference) error { - var errs []error - - for _, ref := range remoteRefs { - if ref == nil { - continue - } - config := &unstructured.Unstructured{} - config.SetKind(ref.Kind) - config.SetAPIVersion(ref.APIVersion) - config.SetNamespace(ref.Namespace) - config.SetName(ref.Name) - - if err := r.Client.Delete(ctx, config); err != nil && !apierrors.IsNotFound(err) { - errs = append(errs, errors.Wrap(err, "failed to cleanup generated resources after error")) - } - } - - return kerrors.NewAggregate(errs) -} - -func (r *K3sControlPlaneReconciler) generateK3sConfig(ctx context.Context, kcp *infracontrolplanev1.K3sControlPlane, cluster *clusterv1.Cluster, spec *infrabootstrapv1.K3sConfigSpec) (*corev1.ObjectReference, error) { - // Create an owner reference without a controller reference because the owning controller is the machine controller - owner := metav1.OwnerReference{ - APIVersion: infracontrolplanev1.GroupVersion.String(), - Kind: "K3sControlPlane", - Name: kcp.Name, - UID: kcp.UID, - } - - bootstrapConfig := &infrabootstrapv1.K3sConfig{ - ObjectMeta: metav1.ObjectMeta{ - Name: names.SimpleNameGenerator.GenerateName(kcp.Name + "-"), - Namespace: kcp.Namespace, - Labels: k3sCluster.ControlPlaneMachineLabelsForCluster(kcp, cluster.Name), - Annotations: kcp.Spec.MachineTemplate.ObjectMeta.Annotations, - OwnerReferences: []metav1.OwnerReference{owner}, - }, - Spec: *spec, - } - - if err := r.Client.Create(ctx, bootstrapConfig); err != nil { - return nil, errors.Wrap(err, "Failed to create bootstrap configuration") - } - - bootstrapRef := &corev1.ObjectReference{ - APIVersion: bootstrapv1.GroupVersion.String(), - Kind: "K3sConfig", - Name: bootstrapConfig.GetName(), - Namespace: bootstrapConfig.GetNamespace(), - UID: bootstrapConfig.GetUID(), - } - - return bootstrapRef, nil -} - -func (r *K3sControlPlaneReconciler) generateMachine(ctx context.Context, kcp *infracontrolplanev1.K3sControlPlane, cluster *clusterv1.Cluster, infraRef, bootstrapRef *corev1.ObjectReference, failureDomain *string) error { - machine := &clusterv1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: names.SimpleNameGenerator.GenerateName(kcp.Name + "-"), - Namespace: kcp.Namespace, - Labels: k3sCluster.ControlPlaneMachineLabelsForCluster(kcp, cluster.Name), - Annotations: map[string]string{}, - OwnerReferences: []metav1.OwnerReference{ - *metav1.NewControllerRef(kcp, controlplanev1.GroupVersion.WithKind("K3sControlPlane")), - }, - }, - Spec: clusterv1.MachineSpec{ - ClusterName: cluster.Name, - Version: &kcp.Spec.Version, - InfrastructureRef: *infraRef, - Bootstrap: clusterv1.Bootstrap{ - ConfigRef: bootstrapRef, - }, - FailureDomain: failureDomain, - NodeDrainTimeout: kcp.Spec.MachineTemplate.NodeDrainTimeout, - }, - } - if kcp.Spec.MachineTemplate.NodeDeletionTimeout != nil { - machine.Spec.NodeDeletionTimeout = kcp.Spec.MachineTemplate.NodeDeletionTimeout - } - - // Machine's bootstrap config may be missing ClusterConfiguration if it is not the first machine in the control plane. - // We store ClusterConfiguration as annotation here to detect any changes in KCP ClusterConfiguration and rollout the machine if any. - serverConfig, err := json.Marshal(kcp.Spec.K3sConfigSpec.ServerConfiguration) - if err != nil { - return errors.Wrap(err, "failed to marshal cluster configuration") - } - - // Add the annotations from the MachineTemplate. - // Note: we intentionally don't use the map directly to ensure we don't modify the map in KCP. - for k, v := range kcp.Spec.MachineTemplate.ObjectMeta.Annotations { - machine.Annotations[k] = v - } - machine.Annotations[infracontrolplanev1.K3sServerConfigurationAnnotation] = string(serverConfig) - - if err := r.Client.Create(ctx, machine); err != nil { - return errors.Wrap(err, "failed to create machine") - } - return nil -} diff --git a/controlplane/k3s/controllers/k3scontrolplane_controller.go b/controlplane/k3s/controllers/k3scontrolplane_controller.go deleted file mode 100644 index 27500865..00000000 --- a/controlplane/k3s/controllers/k3scontrolplane_controller.go +++ /dev/null @@ -1,593 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "context" - "fmt" - "time" - - "github.com/blang/semver" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - kerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/client-go/tools/record" - "k8s.io/utils/pointer" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/remote" - expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" - "sigs.k8s.io/cluster-api/feature" - "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/cluster-api/util/annotations" - "sigs.k8s.io/cluster-api/util/collections" - "sigs.k8s.io/cluster-api/util/conditions" - "sigs.k8s.io/cluster-api/util/patch" - "sigs.k8s.io/cluster-api/util/predicates" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/source" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" - k3sCluster "github.com/kubesphere/kubekey/v3/controlplane/k3s/pkg/cluster" - "github.com/kubesphere/kubekey/v3/util/secret" -) - -// K3sControlPlaneReconciler reconciles a K3sControlPlane object -type K3sControlPlaneReconciler struct { - client.Client - Scheme *runtime.Scheme - APIReader client.Reader - controller controller.Controller - recorder record.EventRecorder - Tracker *remote.ClusterCacheTracker - - // WatchFilterValue is the label value used to filter events prior to reconciliation. - WatchFilterValue string - - managementCluster k3sCluster.ManagementCluster - managementClusterUncached k3sCluster.ManagementCluster -} - -// SetupWithManager sets up the controller with the Manager. -func (r *K3sControlPlaneReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { - c, err := ctrl.NewControllerManagedBy(mgr). - For(&infracontrolplanev1.K3sControlPlane{}). - Owns(&clusterv1.Machine{}). - WithOptions(options). - WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue)). - Build(r) - if err != nil { - return errors.Wrap(err, "failed setting up with a controller manager") - } - - err = c.Watch( - &source.Kind{Type: &clusterv1.Cluster{}}, - handler.EnqueueRequestsFromMapFunc(r.ClusterToK3sControlPlane), - predicates.All(ctrl.LoggerFrom(ctx), - predicates.ResourceHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue), - predicates.ClusterUnpausedAndInfrastructureReady(ctrl.LoggerFrom(ctx)), - ), - ) - if err != nil { - return errors.Wrap(err, "failed adding Watch for Clusters to controller manager") - } - - r.controller = c - r.recorder = mgr.GetEventRecorderFor("k3s-control-plane-controller") - - if r.managementCluster == nil { - if r.Tracker == nil { - return errors.New("cluster cache tracker is nil, cannot create the internal management cluster resource") - } - r.managementCluster = &k3sCluster.Management{ - Client: r.Client, - Tracker: r.Tracker, - } - } - - if r.managementClusterUncached == nil { - r.managementClusterUncached = &k3sCluster.Management{Client: mgr.GetAPIReader()} - } - - return nil -} - -// +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;patch -// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch -// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io;bootstrap.cluster.x-k8s.io;controlplane.cluster.x-k8s.io,resources=*,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status,verbs=get;list;watch -// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines;machines/status,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch - -// Reconcile handles K3sControlPlane events. -func (r *K3sControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, retErr error) { - log := ctrl.LoggerFrom(ctx) - - // Fetch the K3sControlPlane instance. - kcp := &infracontrolplanev1.K3sControlPlane{} - if err := r.Client.Get(ctx, req.NamespacedName, kcp); err != nil { - if apierrors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{Requeue: true}, nil - } - - // Fetch the Cluster. - cluster, err := util.GetOwnerCluster(ctx, r.Client, kcp.ObjectMeta) - if err != nil { - log.Error(err, "Failed to retrieve owner Cluster from the API Server") - return ctrl.Result{}, err - } - if cluster == nil { - log.Info("Cluster Controller has not yet set OwnerRef") - return ctrl.Result{}, nil - } - log = log.WithValues("cluster", cluster.Name) - - if annotations.IsPaused(cluster, kcp) { - log.Info("Reconciliation is paused for this object") - return ctrl.Result{}, nil - } - - // Initialize the patch helper. - patchHelper, err := patch.NewHelper(kcp, r.Client) - if err != nil { - log.Error(err, "Failed to configure the patch helper") - return ctrl.Result{Requeue: true}, nil - } - - // Add finalizer first if not exist to avoid the race condition between init and delete - if !controllerutil.ContainsFinalizer(kcp, infracontrolplanev1.K3sControlPlaneFinalizer) { - controllerutil.AddFinalizer(kcp, infracontrolplanev1.K3sControlPlaneFinalizer) - - // patch and return right away instead of reusing the main defer, - // because the main defer may take too much time to get cluster status - // Patch ObservedGeneration only if the reconciliation completed successfully - patchOpts := []patch.Option{patch.WithStatusObservedGeneration{}} - if err := patchHelper.Patch(ctx, kcp, patchOpts...); err != nil { - log.Error(err, "Failed to patch K3sControlPlane to add finalizer") - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil - } - - defer func() { - // Always attempt to update status. - if err := r.updateStatus(ctx, kcp, cluster); err != nil { - var connFailure *k3sCluster.RemoteClusterConnectionError - if errors.As(err, &connFailure) { - log.Info("Could not connect to workload cluster to fetch status", "err", err.Error()) - } else { - log.Error(err, "Failed to update K3sControlPlane Status") - retErr = kerrors.NewAggregate([]error{retErr, err}) - } - } - - // Always attempt to Patch the K3sControlPlane object and status after each reconciliation. - if err := patchK3sControlPlane(ctx, patchHelper, kcp); err != nil { - log.Error(err, "Failed to patch K3sControlPlane") - retErr = kerrors.NewAggregate([]error{retErr, err}) - } - - // TODO: remove this as soon as we have a proper remote cluster cache in place. - // Make KCP to requeue in case status is not ready, so we can check for node status without waiting for a full resync (by default 10 minutes). - // Only requeue if we are not going in exponential backoff due to error, or if we are not already re-queueing, or if the object has a deletion timestamp. - if retErr == nil && !res.Requeue && res.RequeueAfter <= 0 && kcp.ObjectMeta.DeletionTimestamp.IsZero() { - if !kcp.Status.Ready { - res = ctrl.Result{RequeueAfter: 20 * time.Second} - } - } - }() - - if !kcp.ObjectMeta.DeletionTimestamp.IsZero() { - // Handle deletion reconciliation loop. - return r.reconcileDelete(ctx, cluster, kcp) - } - - // Handle normal reconciliation loop. - return r.reconcile(ctx, cluster, kcp) -} - -func patchK3sControlPlane(ctx context.Context, patchHelper *patch.Helper, kcp *infracontrolplanev1.K3sControlPlane) error { - // Always update the readyCondition by summarizing the state of other conditions. - conditions.SetSummary(kcp, - conditions.WithConditions( - infracontrolplanev1.MachinesCreatedCondition, - infracontrolplanev1.MachinesSpecUpToDateCondition, - infracontrolplanev1.ResizedCondition, - infracontrolplanev1.MachinesReadyCondition, - infracontrolplanev1.AvailableCondition, - infracontrolplanev1.CertificatesAvailableCondition, - ), - ) - - // Patch the object, ignoring conflicts on the conditions owned by this controller. - return patchHelper.Patch( - ctx, - kcp, - patch.WithOwnedConditions{Conditions: []clusterv1.ConditionType{ - infracontrolplanev1.MachinesCreatedCondition, - clusterv1.ReadyCondition, - infracontrolplanev1.MachinesSpecUpToDateCondition, - infracontrolplanev1.ResizedCondition, - infracontrolplanev1.MachinesReadyCondition, - infracontrolplanev1.AvailableCondition, - infracontrolplanev1.CertificatesAvailableCondition, - }}, - patch.WithStatusObservedGeneration{}, - ) -} - -// reconcile handles K3sControlPlane reconciliation. -func (r *K3sControlPlaneReconciler) reconcile(ctx context.Context, cluster *clusterv1.Cluster, kcp *infracontrolplanev1.K3sControlPlane) (res ctrl.Result, retErr error) { - log := ctrl.LoggerFrom(ctx, "cluster", cluster.Name) - log.Info("Reconcile K3sControlPlane") - - // Make sure to reconcile the external infrastructure reference. - if err := r.reconcileExternalReference(ctx, cluster, &kcp.Spec.MachineTemplate.InfrastructureRef); err != nil { - return ctrl.Result{}, err - } - - // Wait for the cluster infrastructure to be ready before creating machines - if !cluster.Status.InfrastructureReady { - log.Info("Cluster infrastructure is not ready yet") - return ctrl.Result{}, nil - } - - // Generate Cluster Certificates if needed - certificates := secret.NewCertificatesForInitialControlPlane() - controllerRef := metav1.NewControllerRef(kcp, infracontrolplanev1.GroupVersion.WithKind("K3sControlPlane")) - if err := certificates.LookupOrGenerate(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil { - log.Error(err, "unable to lookup or create cluster certificates") - conditions.MarkFalse(kcp, infracontrolplanev1.CertificatesAvailableCondition, infracontrolplanev1.CertificatesGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) - return ctrl.Result{}, err - } - conditions.MarkTrue(kcp, infracontrolplanev1.CertificatesAvailableCondition) - - // If ControlPlaneEndpoint is not set, return early - if !cluster.Spec.ControlPlaneEndpoint.IsValid() { - log.Info("Cluster does not yet have a ControlPlaneEndpoint defined") - return ctrl.Result{}, nil - } - - // Generate Cluster Kubeconfig if needed - if result, err := r.reconcileKubeconfig(ctx, cluster, kcp); !result.IsZero() || err != nil { - if err != nil { - log.Error(err, "failed to reconcile Kubeconfig") - } - return result, err - } - - controlPlaneMachines, err := r.managementClusterUncached.GetMachinesForCluster(ctx, cluster, collections.ControlPlaneMachines(cluster.Name)) - if err != nil { - log.Error(err, "failed to retrieve control plane machines for cluster") - return ctrl.Result{}, err - } - - adoptableMachines := controlPlaneMachines.Filter(collections.AdoptableControlPlaneMachines(cluster.Name)) - if len(adoptableMachines) > 0 { - // We adopt the Machines and then wait for the update event for the ownership reference to re-queue them so the cache is up-to-date - err = r.adoptMachines(ctx, kcp, adoptableMachines, cluster) - return ctrl.Result{}, err - } - - ownedMachines := controlPlaneMachines.Filter(collections.OwnedMachines(kcp)) - if len(ownedMachines) != len(controlPlaneMachines) { - log.Info("Not all control plane machines are owned by this K3sControlPlane, refusing to operate in mixed management mode") - return ctrl.Result{}, nil - } - - controlPlane, err := k3sCluster.NewControlPlane(ctx, r.Client, cluster, kcp, ownedMachines) - if err != nil { - log.Error(err, "failed to initialize control plane") - return ctrl.Result{}, err - } - - // Aggregate the operational state of all the machines; while aggregating we are adding the - // source ref (reason@machine/name) so the problem can be easily tracked down to its source machine. - conditions.SetAggregate(controlPlane.KCP, infracontrolplanev1.MachinesReadyCondition, ownedMachines.ConditionGetters(), conditions.AddSourceRef(), conditions.WithStepCounterIf(false)) - - // Updates conditions reporting the status of static pods and the status of the etcd cluster. - // NOTE: Conditions reporting KCP operation progress like e.g. Resized or SpecUpToDate are inlined with the rest of the execution. - if result, err := r.reconcileControlPlaneConditions(ctx, controlPlane); err != nil || !result.IsZero() { - return result, err - } - - // Reconcile unhealthy machines by triggering deletion and requeue if it is considered safe to remediate, - // otherwise continue with the other KCP operations. - //if result, err := r.reconcileUnhealthyMachines(ctx, controlPlane); err != nil || !result.IsZero() { - // return result, err - //} - - // Control plane machines rollout due to configuration changes (e.g. upgrades) takes precedence over other operations. - needRollout := controlPlane.MachinesNeedingRollout() - switch { - case len(needRollout) > 0: - log.Info("Rolling out Control Plane machines", "needRollout", needRollout.Names()) - conditions.MarkFalse(controlPlane.KCP, infracontrolplanev1.MachinesSpecUpToDateCondition, infracontrolplanev1.RollingUpdateInProgressReason, clusterv1.ConditionSeverityWarning, "Rolling %d replicas with outdated spec (%d replicas up to date)", len(needRollout), len(controlPlane.Machines)-len(needRollout)) - return r.upgradeControlPlane(ctx, cluster, kcp, controlPlane, needRollout) - default: - // make sure last upgrade operation is marked as completed. - // NOTE: we are checking the condition already exists in order to avoid to set this condition at the first - // reconciliation/before a rolling upgrade actually starts. - if conditions.Has(controlPlane.KCP, infracontrolplanev1.MachinesSpecUpToDateCondition) { - conditions.MarkTrue(controlPlane.KCP, infracontrolplanev1.MachinesSpecUpToDateCondition) - } - } - - // If we've made it this far, we can assume that all ownedMachines are up to date - numMachines := len(ownedMachines) - desiredReplicas := int(*kcp.Spec.Replicas) - - switch { - // We are creating the first replica - case numMachines < desiredReplicas && numMachines == 0: - // Create new Machine w/ init - log.Info("Initializing control plane", "Desired", desiredReplicas, "Existing", numMachines) - conditions.MarkFalse(controlPlane.KCP, infracontrolplanev1.AvailableCondition, infracontrolplanev1.WaitingForKubeadmInitReason, clusterv1.ConditionSeverityInfo, "") - return r.initializeControlPlane(ctx, cluster, kcp, controlPlane) - // We are scaling up - case numMachines < desiredReplicas && numMachines > 0: - // Create a new Machine w/ join - log.Info("Scaling up control plane", "Desired", desiredReplicas, "Existing", numMachines) - return r.scaleUpControlPlane(ctx, cluster, kcp, controlPlane) - // We are scaling down - case numMachines > desiredReplicas: - log.Info("Scaling down control plane", "Desired", desiredReplicas, "Existing", numMachines) - // The last parameter (i.e. machines needing to be rolled out) should always be empty here. - return r.scaleDownControlPlane(ctx, cluster, kcp, controlPlane, collections.Machines{}) - } - - return ctrl.Result{}, nil -} - -// reconcileDelete handles K3sControlPlane deletion. -// The implementation does not take non-control plane workloads into consideration. This may or may not change in the future. -// Please see https://github.com/kubernetes-sigs/cluster-api/issues/2064. -func (r *K3sControlPlaneReconciler) reconcileDelete(ctx context.Context, cluster *clusterv1.Cluster, kcp *infracontrolplanev1.K3sControlPlane) (ctrl.Result, error) { - log := ctrl.LoggerFrom(ctx, "cluster", cluster.Name) - log.Info("Reconcile K3sControlPlane deletion") - - // Gets all machines, not just control plane machines. - allMachines, err := r.managementCluster.GetMachinesForCluster(ctx, cluster) - if err != nil { - return ctrl.Result{}, err - } - ownedMachines := allMachines.Filter(collections.OwnedMachines(kcp)) - - // If no control plane machines remain, remove the finalizer - if len(ownedMachines) == 0 { - controllerutil.RemoveFinalizer(kcp, infracontrolplanev1.K3sControlPlaneFinalizer) - return ctrl.Result{}, nil - } - - controlPlane, err := k3sCluster.NewControlPlane(ctx, r.Client, cluster, kcp, ownedMachines) - if err != nil { - log.Error(err, "failed to initialize control plane") - return ctrl.Result{}, err - } - - // Updates conditions reporting the status of static pods and the status of the etcd cluster. - // NOTE: Ignoring failures given that we are deleting - if _, err := r.reconcileControlPlaneConditions(ctx, controlPlane); err != nil { - log.Info("failed to reconcile conditions", "error", err.Error()) - } - - // Aggregate the operational state of all the machines; while aggregating we are adding the - // source ref (reason@machine/name) so the problem can be easily tracked down to its source machine. - // However, during delete we are hiding the counter (1 of x) because it does not make sense given that - // all the machines are deleted in parallel. - conditions.SetAggregate(kcp, infracontrolplanev1.MachinesReadyCondition, ownedMachines.ConditionGetters(), conditions.AddSourceRef(), conditions.WithStepCounterIf(false)) - - allMachinePools := &expv1.MachinePoolList{} - // Get all machine pools. - if feature.Gates.Enabled(feature.MachinePool) { - allMachinePools, err = r.managementCluster.GetMachinePoolsForCluster(ctx, cluster) - if err != nil { - return ctrl.Result{}, err - } - } - // Verify that only control plane machines remain - if len(allMachines) != len(ownedMachines) || len(allMachinePools.Items) != 0 { - log.Info("Waiting for worker nodes to be deleted first") - conditions.MarkFalse(kcp, infracontrolplanev1.ResizedCondition, clusterv1.DeletingReason, clusterv1.ConditionSeverityInfo, "Waiting for worker nodes to be deleted first") - return ctrl.Result{RequeueAfter: deleteRequeueAfter}, nil - } - - // Delete control plane machines in parallel - machinesToDelete := ownedMachines.Filter(collections.Not(collections.HasDeletionTimestamp)) - var errs []error - for i := range machinesToDelete { - m := machinesToDelete[i] - logger := log.WithValues("machine", m) - if err := r.Client.Delete(ctx, machinesToDelete[i]); err != nil && !apierrors.IsNotFound(err) { - logger.Error(err, "Failed to cleanup owned machine") - errs = append(errs, err) - } - } - if len(errs) > 0 { - err := kerrors.NewAggregate(errs) - r.recorder.Eventf(kcp, corev1.EventTypeWarning, "FailedDelete", - "Failed to delete control plane Machines for cluster %s/%s control plane: %v", cluster.Namespace, cluster.Name, err) - return ctrl.Result{}, err - } - conditions.MarkFalse(kcp, infracontrolplanev1.ResizedCondition, clusterv1.DeletingReason, clusterv1.ConditionSeverityInfo, "") - return ctrl.Result{RequeueAfter: deleteRequeueAfter}, nil -} - -func (r *K3sControlPlaneReconciler) adoptMachines(ctx context.Context, kcp *infracontrolplanev1.K3sControlPlane, machines collections.Machines, cluster *clusterv1.Cluster) error { - // We do an uncached full quorum read against the KCP to avoid re-adopting Machines the garbage collector just intentionally orphaned - // See https://github.com/kubernetes/kubernetes/issues/42639 - uncached := infracontrolplanev1.K3sControlPlane{} - err := r.managementClusterUncached.Get(ctx, client.ObjectKey{Namespace: kcp.Namespace, Name: kcp.Name}, &uncached) - if err != nil { - return errors.Wrapf(err, "failed to check whether %v/%v was deleted before adoption", kcp.GetNamespace(), kcp.GetName()) - } - if !uncached.DeletionTimestamp.IsZero() { - return errors.Errorf("%v/%v has just been deleted at %v", kcp.GetNamespace(), kcp.GetName(), kcp.GetDeletionTimestamp()) - } - - kcpVersion, err := semver.ParseTolerant(kcp.Spec.Version) - if err != nil { - return errors.Wrapf(err, "failed to parse kubernetes version %q", kcp.Spec.Version) - } - - for _, m := range machines { - ref := m.Spec.Bootstrap.ConfigRef - - // TODO instead of returning error here, we should instead Event and add a watch on potentially adoptable Machines - if ref == nil || ref.Kind != "K3sConfig" { - return errors.Errorf("unable to adopt Machine %v/%v: expected a ConfigRef of kind K3sConfig but instead found %v", m.Namespace, m.Name, ref) - } - - // TODO instead of returning error here, we should instead Event and add a watch on potentially adoptable Machines - if ref.Namespace != "" && ref.Namespace != kcp.Namespace { - return errors.Errorf("could not adopt resources from K3sConfig %v/%v: cannot adopt across namespaces", ref.Namespace, ref.Name) - } - - if m.Spec.Version == nil { - // if the machine's version is not immediately apparent, assume the operator knows what they're doing - continue - } - - machineVersion, err := semver.ParseTolerant(*m.Spec.Version) - if err != nil { - return errors.Wrapf(err, "failed to parse kubernetes version %q", *m.Spec.Version) - } - - if !util.IsSupportedVersionSkew(kcpVersion, machineVersion) { - r.recorder.Eventf(kcp, corev1.EventTypeWarning, "AdoptionFailed", "Could not adopt Machine %s/%s: its version (%q) is outside supported +/- one minor version skew from KCP's (%q)", m.Namespace, m.Name, *m.Spec.Version, kcp.Spec.Version) - // avoid returning an error here so we don't cause the KCP controller to spin until the operator clarifies their intent - return nil - } - } - - for _, m := range machines { - ref := m.Spec.Bootstrap.ConfigRef - cfg := &infrabootstrapv1.K3sConfig{} - - if err := r.Client.Get(ctx, client.ObjectKey{Name: ref.Name, Namespace: kcp.Namespace}, cfg); err != nil { - return err - } - - if err := r.adoptOwnedSecrets(ctx, kcp, cfg, cluster.Name); err != nil { - return err - } - - patchHelper, err := patch.NewHelper(m, r.Client) - if err != nil { - return err - } - - if err := controllerutil.SetControllerReference(kcp, m, r.Client.Scheme()); err != nil { - return err - } - - // Note that ValidateOwnerReferences() will reject this patch if another - // OwnerReference exists with controller=true. - if err := patchHelper.Patch(ctx, m); err != nil { - return err - } - } - return nil -} - -func (r *K3sControlPlaneReconciler) adoptOwnedSecrets(ctx context.Context, kcp *infracontrolplanev1.K3sControlPlane, currentOwner *infrabootstrapv1.K3sConfig, clusterName string) error { - secrets := corev1.SecretList{} - if err := r.Client.List(ctx, &secrets, client.InNamespace(kcp.Namespace), client.MatchingLabels{clusterv1.ClusterLabelName: clusterName}); err != nil { - return errors.Wrap(err, "error finding secrets for adoption") - } - - for i := range secrets.Items { - s := secrets.Items[i] - if !util.IsOwnedByObject(&s, currentOwner) { - continue - } - // avoid taking ownership of the bootstrap data secret - if currentOwner.Status.DataSecretName != nil && s.Name == *currentOwner.Status.DataSecretName { - continue - } - - ss := s.DeepCopy() - - ss.SetOwnerReferences(util.ReplaceOwnerRef(ss.GetOwnerReferences(), currentOwner, metav1.OwnerReference{ - APIVersion: infracontrolplanev1.GroupVersion.String(), - Kind: "K3sControlPlane", - Name: kcp.Name, - UID: kcp.UID, - Controller: pointer.Bool(true), - BlockOwnerDeletion: pointer.Bool(true), - })) - - if err := r.Client.Update(ctx, ss); err != nil { - return errors.Wrapf(err, "error changing secret %v ownership from K3sConfig/%v to K3sControlPlane/%v", s.Name, currentOwner.GetName(), kcp.Name) - } - } - - return nil -} - -// reconcileControlPlaneConditions is responsible of reconciling conditions reporting the status of static pods and -// the status of the etcd cluster. -func (r *K3sControlPlaneReconciler) reconcileControlPlaneConditions(ctx context.Context, controlPlane *k3sCluster.ControlPlane) (ctrl.Result, error) { - // If the cluster is not yet initialized, there is no way to connect to the workload cluster and fetch information - // for updating conditions. Return early. - if !controlPlane.KCP.Status.Initialized { - return ctrl.Result{}, nil - } - - workloadCluster, err := r.managementCluster.GetWorkloadCluster(ctx, util.ObjectKey(controlPlane.Cluster)) - if err != nil { - return ctrl.Result{}, errors.Wrap(err, "cannot get remote client to workload cluster") - } - - // Update conditions status - workloadCluster.UpdateAgentConditions(ctx, controlPlane) - workloadCluster.UpdateEtcdConditions(ctx, controlPlane) - - // Patch machines with the updated conditions. - if err := controlPlane.PatchMachines(ctx); err != nil { - return ctrl.Result{}, err - } - - // KCP will be patched at the end of Reconcile to reflect updated conditions, so we can return now. - return ctrl.Result{}, nil -} - -// ClusterToK3sControlPlane is a handler.ToRequestsFunc to be used to enqueue requests for reconciliation -// for K3sControlPlane based on updates to a Cluster. -func (r *K3sControlPlaneReconciler) ClusterToK3sControlPlane(o client.Object) []ctrl.Request { - c, ok := o.(*clusterv1.Cluster) - if !ok { - panic(fmt.Sprintf("Expected a Cluster but got a %T", o)) - } - - controlPlaneRef := c.Spec.ControlPlaneRef - if controlPlaneRef != nil && controlPlaneRef.Kind == "K3sControlPlane" { - return []ctrl.Request{{NamespacedName: client.ObjectKey{Namespace: controlPlaneRef.Namespace, Name: controlPlaneRef.Name}}} - } - - return nil -} diff --git a/controlplane/k3s/controllers/scale.go b/controlplane/k3s/controllers/scale.go deleted file mode 100644 index 02dbb2e4..00000000 --- a/controlplane/k3s/controllers/scale.go +++ /dev/null @@ -1,206 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package controllers - -import ( - "context" - "strings" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - kerrors "k8s.io/apimachinery/pkg/util/errors" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/util/collections" - "sigs.k8s.io/cluster-api/util/conditions" - ctrl "sigs.k8s.io/controller-runtime" - - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" - k3sCluster "github.com/kubesphere/kubekey/v3/controlplane/k3s/pkg/cluster" -) - -func (r *K3sControlPlaneReconciler) initializeControlPlane(ctx context.Context, cluster *clusterv1.Cluster, kcp *infracontrolplanev1.K3sControlPlane, controlPlane *k3sCluster.ControlPlane) (ctrl.Result, error) { - logger := controlPlane.Logger() - - // Perform an uncached read of all the owned machines. This check is in place to make sure - // that the controller cache is not misbehaving and we end up initializing the cluster more than once. - ownedMachines, err := r.managementClusterUncached.GetMachinesForCluster(ctx, cluster, collections.OwnedMachines(kcp)) - if err != nil { - logger.Error(err, "failed to perform an uncached read of control plane machines for cluster") - return ctrl.Result{}, err - } - if len(ownedMachines) > 0 { - return ctrl.Result{}, errors.Errorf( - "control plane has already been initialized, found %d owned machine for cluster %s/%s: controller cache or management cluster is misbehaving", - len(ownedMachines), cluster.Namespace, cluster.Name, - ) - } - - bootstrapSpec := controlPlane.InitialControlPlaneConfig() - fd := controlPlane.NextFailureDomainForScaleUp() - if err := r.cloneConfigsAndGenerateMachine(ctx, cluster, kcp, bootstrapSpec, fd); err != nil { - logger.Error(err, "Failed to create initial control plane Machine") - r.recorder.Eventf(kcp, corev1.EventTypeWarning, "FailedInitialization", "Failed to create initial control plane Machine for cluster %s/%s control plane: %v", cluster.Namespace, cluster.Name, err) - return ctrl.Result{}, err - } - - // Requeue the control plane, in case there are additional operations to perform - return ctrl.Result{Requeue: true}, nil -} - -func (r *K3sControlPlaneReconciler) scaleUpControlPlane(ctx context.Context, cluster *clusterv1.Cluster, kcp *infracontrolplanev1.K3sControlPlane, controlPlane *k3sCluster.ControlPlane) (ctrl.Result, error) { - logger := controlPlane.Logger() - - // Run preflight checks to ensure that the control plane is stable before proceeding with a scale up/scale down operation; if not, wait. - if result, err := r.preflightChecks(ctx, controlPlane); err != nil || !result.IsZero() { - return result, err - } - - // Create the bootstrap configuration - bootstrapSpec := controlPlane.JoinControlPlaneConfig() - fd := controlPlane.NextFailureDomainForScaleUp() - if err := r.cloneConfigsAndGenerateMachine(ctx, cluster, kcp, bootstrapSpec, fd); err != nil { - logger.Error(err, "Failed to create additional control plane Machine") - r.recorder.Eventf(kcp, corev1.EventTypeWarning, "FailedScaleUp", "Failed to create additional control plane Machine for cluster %s/%s control plane: %v", cluster.Namespace, cluster.Name, err) - return ctrl.Result{}, err - } - - // Requeue the control plane, in case there are other operations to perform - return ctrl.Result{Requeue: true}, nil -} - -func (r *K3sControlPlaneReconciler) scaleDownControlPlane( - ctx context.Context, - cluster *clusterv1.Cluster, - kcp *infracontrolplanev1.K3sControlPlane, - controlPlane *k3sCluster.ControlPlane, - outdatedMachines collections.Machines, -) (ctrl.Result, error) { - logger := controlPlane.Logger() - - // Pick the Machine that we should scale down. - machineToDelete, err := selectMachineForScaleDown(controlPlane, outdatedMachines) - if err != nil { - return ctrl.Result{}, errors.Wrap(err, "failed to select machine for scale down") - } - - // Run preflight checks ensuring the control plane is stable before proceeding with a scale up/scale down operation; if not, wait. - // Given that we're scaling down, we can exclude the machineToDelete from the preflight checks. - if result, err := r.preflightChecks(ctx, controlPlane, machineToDelete); err != nil || !result.IsZero() { - return result, err - } - - if machineToDelete == nil { - logger.Info("Failed to pick control plane Machine to delete") - return ctrl.Result{}, errors.New("failed to pick control plane Machine to delete") - } - - logger = logger.WithValues("machine", machineToDelete.Name) - if err := r.Client.Delete(ctx, machineToDelete); err != nil && !apierrors.IsNotFound(err) { - logger.Error(err, "Failed to delete control plane machine") - r.recorder.Eventf(kcp, corev1.EventTypeWarning, "FailedScaleDown", - "Failed to delete control plane Machine %s for cluster %s/%s control plane: %v", machineToDelete.Name, cluster.Namespace, cluster.Name, err) - return ctrl.Result{}, err - } - - // Requeue the control plane, in case there are additional operations to perform - return ctrl.Result{Requeue: true}, nil -} - -// preflightChecks checks if the control plane is stable before proceeding with a scale up/scale down operation, -// where stable means that: -// - There are no machine deletion in progress -// - All the health conditions on KCP are true. -// - All the health conditions on the control plane machines are true. -// If the control plane is not passing preflight checks, it requeue. -// -// NOTE: this func uses KCP conditions, it is required to call reconcileControlPlaneConditions before this. -func (r *K3sControlPlaneReconciler) preflightChecks(_ context.Context, controlPlane *k3sCluster.ControlPlane, excludeFor ...*clusterv1.Machine) (ctrl.Result, error) { //nolint:unparam - logger := controlPlane.Logger() - - // If there is no KCP-owned control-plane machines, then control-plane has not been initialized yet, - // so it is considered ok to proceed. - if controlPlane.Machines.Len() == 0 { - return ctrl.Result{}, nil - } - - // If there are deleting machines, wait for the operation to complete. - if controlPlane.HasDeletingMachine() { - logger.Info("Waiting for machines to be deleted", "Machines", strings.Join(controlPlane.Machines.Filter(collections.HasDeletionTimestamp).Names(), ", ")) - return ctrl.Result{RequeueAfter: deleteRequeueAfter}, nil - } - - // Check machine health conditions; if there are conditions with False or Unknown, then wait. - allMachineHealthConditions := []clusterv1.ConditionType{ - infracontrolplanev1.MachineAgentHealthyCondition, - } - var machineErrors []error - -loopmachines: - for _, machine := range controlPlane.Machines { - for _, excluded := range excludeFor { - // If this machine should be excluded from the individual - // health check, continue the out loop. - if machine.Name == excluded.Name { - continue loopmachines - } - } - - for _, condition := range allMachineHealthConditions { - if err := preflightCheckCondition("machine", machine, condition); err != nil { - machineErrors = append(machineErrors, err) - } - } - } - if len(machineErrors) > 0 { - aggregatedError := kerrors.NewAggregate(machineErrors) - r.recorder.Eventf(controlPlane.KCP, corev1.EventTypeWarning, "ControlPlaneUnhealthy", - "Waiting for control plane to pass preflight checks to continue reconciliation: %v", aggregatedError) - logger.Info("Waiting for control plane to pass preflight checks", "failures", aggregatedError.Error()) - - return ctrl.Result{RequeueAfter: preflightFailedRequeueAfter}, nil - } - - return ctrl.Result{}, nil -} - -func preflightCheckCondition(kind string, obj conditions.Getter, condition clusterv1.ConditionType) error { - c := conditions.Get(obj, condition) - if c == nil { - return errors.Errorf("%s %s does not have %s condition", kind, obj.GetName(), condition) - } - if c.Status == corev1.ConditionFalse { - return errors.Errorf("%s %s reports %s condition is false (%s, %s)", kind, obj.GetName(), condition, c.Severity, c.Message) - } - if c.Status == corev1.ConditionUnknown { - return errors.Errorf("%s %s reports %s condition is unknown (%s)", kind, obj.GetName(), condition, c.Message) - } - return nil -} - -func selectMachineForScaleDown(controlPlane *k3sCluster.ControlPlane, outdatedMachines collections.Machines) (*clusterv1.Machine, error) { - machines := controlPlane.Machines - switch { - case controlPlane.MachineWithDeleteAnnotation(outdatedMachines).Len() > 0: - machines = controlPlane.MachineWithDeleteAnnotation(outdatedMachines) - case controlPlane.MachineWithDeleteAnnotation(machines).Len() > 0: - machines = controlPlane.MachineWithDeleteAnnotation(machines) - case outdatedMachines.Len() > 0: - machines = outdatedMachines - } - return controlPlane.MachineInFailureDomainWithMostMachines(machines) -} diff --git a/controlplane/k3s/controllers/status.go b/controlplane/k3s/controllers/status.go deleted file mode 100644 index b15b94f4..00000000 --- a/controlplane/k3s/controllers/status.go +++ /dev/null @@ -1,119 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package controllers - -import ( - "context" - - "github.com/pkg/errors" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/cluster-api/util/collections" - "sigs.k8s.io/cluster-api/util/conditions" - ctrl "sigs.k8s.io/controller-runtime" - - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" - k3sCluster "github.com/kubesphere/kubekey/v3/controlplane/k3s/pkg/cluster" -) - -// updateStatus is called after every reconcilitation loop in a defer statement to always make sure we have the -// resource status subresourcs up-to-date. -func (r *K3sControlPlaneReconciler) updateStatus(ctx context.Context, kcp *infracontrolplanev1.K3sControlPlane, cluster *clusterv1.Cluster) error { - log := ctrl.LoggerFrom(ctx, "cluster", cluster.Name) - - selector := collections.ControlPlaneSelectorForCluster(cluster.Name) - // Copy label selector to its status counterpart in string format. - // This is necessary for CRDs including scale subresources. - kcp.Status.Selector = selector.String() - - ownedMachines, err := r.managementCluster.GetMachinesForCluster(ctx, cluster, collections.OwnedMachines(kcp)) - if err != nil { - return errors.Wrap(err, "failed to get list of owned machines") - } - - controlPlane, err := k3sCluster.NewControlPlane(ctx, r.Client, cluster, kcp, ownedMachines) - if err != nil { - log.Error(err, "failed to initialize control plane") - return err - } - kcp.Status.UpdatedReplicas = int32(len(controlPlane.UpToDateMachines())) - - replicas := int32(len(ownedMachines)) - desiredReplicas := *kcp.Spec.Replicas - - // set basic data that does not require interacting with the workload cluster - kcp.Status.Replicas = replicas - kcp.Status.ReadyReplicas = 0 - kcp.Status.UnavailableReplicas = replicas - - // Return early if the deletion timestamp is set, because we don't want to try to connect to the workload cluster - // and we don't want to report resize condition (because it is set to deleting into reconcile delete). - if !kcp.DeletionTimestamp.IsZero() { - return nil - } - - machinesWithHealthAPIServer := ownedMachines.Filter(collections.HealthyAPIServer()) - lowestVersion := machinesWithHealthAPIServer.LowestVersion() - if lowestVersion != nil { - kcp.Status.Version = lowestVersion - } - - switch { - // We are scaling up - case replicas < desiredReplicas: - conditions.MarkFalse(kcp, infracontrolplanev1.ResizedCondition, infracontrolplanev1.ScalingUpReason, clusterv1.ConditionSeverityWarning, "Scaling up control plane to %d replicas (actual %d)", desiredReplicas, replicas) - // We are scaling down - case replicas > desiredReplicas: - conditions.MarkFalse(kcp, infracontrolplanev1.ResizedCondition, infracontrolplanev1.ScalingDownReason, clusterv1.ConditionSeverityWarning, "Scaling down control plane to %d replicas (actual %d)", desiredReplicas, replicas) - - // This means that there was no error in generating the desired number of machine objects - conditions.MarkTrue(kcp, infracontrolplanev1.MachinesCreatedCondition) - default: - // make sure last resize operation is marked as completed. - // NOTE: we are checking the number of machines ready so we report resize completed only when the machines - // are actually provisioned (vs reporting completed immediately after the last machine object is created). - readyMachines := ownedMachines.Filter(collections.IsReady()) - if int32(len(readyMachines)) == replicas { - conditions.MarkTrue(kcp, infracontrolplanev1.ResizedCondition) - } - - // This means that there was no error in generating the desired number of machine objects - conditions.MarkTrue(kcp, infracontrolplanev1.MachinesCreatedCondition) - } - - workloadCluster, err := r.managementCluster.GetWorkloadCluster(ctx, util.ObjectKey(cluster)) - if err != nil { - return errors.Wrap(err, "failed to create remote cluster client") - } - status, err := workloadCluster.ClusterStatus(ctx) - if err != nil { - return err - } - - log.Info("ClusterStatus", "workload", status) - - kcp.Status.ReadyReplicas = status.ReadyNodes - kcp.Status.UnavailableReplicas = replicas - status.ReadyNodes - - if kcp.Status.ReadyReplicas > 0 { - kcp.Status.Ready = true - kcp.Status.Initialized = true - conditions.MarkTrue(kcp, infracontrolplanev1.AvailableCondition) - } - - return nil -} diff --git a/controlplane/k3s/controllers/suite_test.go b/controlplane/k3s/controllers/suite_test.go deleted file mode 100644 index 436d11e4..00000000 --- a/controlplane/k3s/controllers/suite_test.go +++ /dev/null @@ -1,64 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -//// These tests use Ginkgo (BDD-style Go testing framework). Refer to -//// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. -// -//var cfg *rest.Config -//var k8sClient client.Client -//var testEnv *envtest.Environment -// -//func TestAPIs(t *testing.T) { -// RegisterFailHandler(Fail) -// -// RunSpecsWithDefaultAndCustomReporters(t, -// "Controller Suite", -// []Reporter{printer.NewlineReporter{}}) -//} -// -//var _ = BeforeSuite(func() { -// logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) -// -// By("bootstrapping test environment") -// testEnv = &envtest.Environment{ -// CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, -// ErrorIfCRDPathMissing: true, -// } -// -// var err error -// // cfg is defined in this file globally. -// cfg, err = testEnv.Start() -// Expect(err).NotTo(HaveOccurred()) -// Expect(cfg).NotTo(BeNil()) -// -// err = controlplanev1beta1.AddToScheme(scheme.Scheme) -// Expect(err).NotTo(HaveOccurred()) -// -// //+kubebuilder:scaffold:scheme -// -// k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) -// Expect(err).NotTo(HaveOccurred()) -// Expect(k8sClient).NotTo(BeNil()) -// -//}, 60) -// -//var _ = AfterSuite(func() { -// By("tearing down the test environment") -// err := testEnv.Stop() -// Expect(err).NotTo(HaveOccurred()) -//}) diff --git a/controlplane/k3s/controllers/upgrade.go b/controlplane/k3s/controllers/upgrade.go deleted file mode 100644 index d520119d..00000000 --- a/controlplane/k3s/controllers/upgrade.go +++ /dev/null @@ -1,58 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package controllers - -import ( - "context" - - "github.com/pkg/errors" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/util/collections" - ctrl "sigs.k8s.io/controller-runtime" - - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" - k3sCluster "github.com/kubesphere/kubekey/v3/controlplane/k3s/pkg/cluster" -) - -func (r *K3sControlPlaneReconciler) upgradeControlPlane( - ctx context.Context, - cluster *clusterv1.Cluster, - kcp *infracontrolplanev1.K3sControlPlane, - controlPlane *k3sCluster.ControlPlane, - machinesRequireUpgrade collections.Machines, -) (ctrl.Result, error) { - logger := controlPlane.Logger() - - if kcp.Spec.RolloutStrategy == nil || kcp.Spec.RolloutStrategy.RollingUpdate == nil { - return ctrl.Result{}, errors.New("rolloutStrategy is not set") - } - - switch kcp.Spec.RolloutStrategy.Type { - case infracontrolplanev1.RollingUpdateStrategyType: - // RolloutStrategy is currently defaulted and validated to be RollingUpdate - // We can ignore MaxUnavailable because we are enforcing health checks before we get here. - maxNodes := *kcp.Spec.Replicas + int32(kcp.Spec.RolloutStrategy.RollingUpdate.MaxSurge.IntValue()) - if int32(controlPlane.Machines.Len()) < maxNodes { - // scaleUp ensures that we don't continue scaling up while waiting for Machines to have NodeRefs - return r.scaleUpControlPlane(ctx, cluster, kcp, controlPlane) - } - return r.scaleDownControlPlane(ctx, cluster, kcp, controlPlane, machinesRequireUpgrade) - default: - logger.Info("RolloutStrategy type is not set to RollingUpdateStrategyType, unable to determine the strategy for rolling out machines") - return ctrl.Result{}, nil - } -} diff --git a/controlplane/k3s/hack/boilerplate.go.txt b/controlplane/k3s/hack/boilerplate.go.txt deleted file mode 100644 index 62802d18..00000000 --- a/controlplane/k3s/hack/boilerplate.go.txt +++ /dev/null @@ -1,15 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ \ No newline at end of file diff --git a/controlplane/k3s/main.go b/controlplane/k3s/main.go deleted file mode 100644 index 6afe3698..00000000 --- a/controlplane/k3s/main.go +++ /dev/null @@ -1,219 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package main -package main - -import ( - "flag" - "fmt" - "math/rand" - "os" - "time" - - "github.com/spf13/pflag" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/klog/v2" - "k8s.io/klog/v2/klogr" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/remote" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/healthz" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" - "github.com/kubesphere/kubekey/v3/controlplane/k3s/controllers" - //+kubebuilder:scaffold:imports -) - -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") -) - -func init() { - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(clusterv1.AddToScheme(scheme)) - utilruntime.Must(infrabootstrapv1.AddToScheme(scheme)) - utilruntime.Must(infracontrolplanev1.AddToScheme(scheme)) - utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) - //+kubebuilder:scaffold:scheme -} - -var ( - metricsBindAddr string - enableLeaderElection bool - leaderElectionLeaseDuration time.Duration - leaderElectionRenewDeadline time.Duration - leaderElectionRetryPeriod time.Duration - watchFilterValue string - watchNamespace string - k3sControlPlaneConcurrency int - syncPeriod time.Duration - webhookPort int - webhookCertDir string - healthAddr string -) - -// InitFlags initializes the flags. -func InitFlags(fs *pflag.FlagSet) { - fs.StringVar(&metricsBindAddr, "metrics-bind-addr", "localhost:8080", - "The address the metric endpoint binds to.") - - fs.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") - - fs.DurationVar(&leaderElectionLeaseDuration, "leader-elect-lease-duration", 1*time.Minute, - "Interval at which non-leader candidates will wait to force acquire leadership (duration string)") - - fs.DurationVar(&leaderElectionRenewDeadline, "leader-elect-renew-deadline", 40*time.Second, - "Duration that the leading controller manager will retry refreshing leadership before giving up (duration string)") - - fs.DurationVar(&leaderElectionRetryPeriod, "leader-elect-retry-period", 5*time.Second, - "Duration the LeaderElector clients should wait between tries of actions (duration string)") - - fs.StringVar(&watchNamespace, "namespace", "", - "Namespace that the controller watches to reconcile cluster-api objects. If unspecified, the controller watches for cluster-api objects across all namespaces.") - - fs.IntVar(&k3sControlPlaneConcurrency, "k3scontrolplane-concurrency", 10, - "Number of kubeadm control planes to process simultaneously") - - fs.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, - "The minimum interval at which watched resources are reconciled (e.g. 15m)") - - fs.StringVar(&watchFilterValue, "watch-filter", "", - fmt.Sprintf("Label value that the controller watches to reconcile cluster-api objects. Label key is always %s. If unspecified, the controller watches for all cluster-api objects.", clusterv1.WatchLabel)) - - fs.IntVar(&webhookPort, "webhook-port", 9443, - "Webhook Server port") - - fs.StringVar(&webhookCertDir, "webhook-cert-dir", "/tmp/k8s-webhook-server/serving-certs/", - "Webhook cert dir, only used when webhook-port is specified.") - - fs.StringVar(&healthAddr, "health-addr", ":9440", - "The address the health endpoint binds to.") -} - -func main() { - klog.InitFlags(nil) - rand.Seed(time.Now().UnixNano()) - - InitFlags(pflag.CommandLine) - pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) - pflag.CommandLine.AddGoFlagSet(flag.CommandLine) - pflag.Parse() - - ctrl.SetLogger(klogr.New()) - - ctx := ctrl.SetupSignalHandler() - - restConfig := ctrl.GetConfigOrDie() - restConfig.UserAgent = remote.DefaultClusterAPIUserAgent("cluster-api-k3s-control-plane-manager") - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsBindAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "k3s-control-plane-manager-leader-election-capkk", - LeaseDuration: &leaderElectionLeaseDuration, - RenewDeadline: &leaderElectionRenewDeadline, - RetryPeriod: &leaderElectionRetryPeriod, - Namespace: watchNamespace, - SyncPeriod: &syncPeriod, - ClientDisableCacheFor: []client.Object{ - &corev1.ConfigMap{}, - &corev1.Secret{}, - }, - Port: webhookPort, - HealthProbeBindAddress: healthAddr, - CertDir: webhookCertDir, - }) - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) - } - - log := ctrl.Log.WithName("remote").WithName("ClusterCacheTracker") - tracker, err := remote.NewClusterCacheTracker(mgr, remote.ClusterCacheTrackerOptions{ - Log: &log, - Indexes: remote.DefaultIndexes, - ClientUncachedObjects: []client.Object{ - &corev1.ConfigMap{}, - &corev1.Secret{}, - &corev1.Pod{}, - &appsv1.Deployment{}, - &appsv1.DaemonSet{}, - }, - }) - if err != nil { - setupLog.Error(err, "unable to create cluster cache tracker") - os.Exit(1) - } - if err := (&remote.ClusterCacheReconciler{ - Client: mgr.GetClient(), - Tracker: tracker, - WatchFilterValue: watchFilterValue, - }).SetupWithManager(ctx, mgr, concurrency(k3sControlPlaneConcurrency)); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "ClusterCacheReconciler") - os.Exit(1) - } - - if err = (&controllers.K3sControlPlaneReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - APIReader: mgr.GetAPIReader(), - Tracker: tracker, - WatchFilterValue: watchFilterValue, - }).SetupWithManager(ctx, mgr, concurrency(k3sControlPlaneConcurrency)); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "K3sControlPlane") - os.Exit(1) - } - if err = (&infracontrolplanev1.K3sControlPlane{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "K3sControlPlane") - os.Exit(1) - } - if err = (&infracontrolplanev1.K3sControlPlaneTemplate{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "K3sControlPlaneTemplate") - os.Exit(1) - } - //+kubebuilder:scaffold:builder - - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") - os.Exit(1) - } - - setupLog.Info("starting manager") - if err := mgr.Start(ctx); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) - } -} - -func concurrency(c int) controller.Options { - return controller.Options{MaxConcurrentReconciles: c} -} diff --git a/controlplane/k3s/pkg/cluster/cluster.go b/controlplane/k3s/pkg/cluster/cluster.go deleted file mode 100644 index d9844081..00000000 --- a/controlplane/k3s/pkg/cluster/cluster.go +++ /dev/null @@ -1,113 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cluster - -import ( - "context" - "time" - - "github.com/pkg/errors" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/remote" - expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" - "sigs.k8s.io/cluster-api/util/collections" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - // K3ssControlPlaneControllerName defines the controller used when creating clients. - K3ssControlPlaneControllerName = "k3s-controlplane-controller" -) - -// ManagementCluster defines all behaviors necessary for something to function as a management cluster. -type ManagementCluster interface { - client.Reader - - GetMachinesForCluster(ctx context.Context, cluster *clusterv1.Cluster, filters ...collections.Func) (collections.Machines, error) - GetMachinePoolsForCluster(ctx context.Context, cluster *clusterv1.Cluster) (*expv1.MachinePoolList, error) - GetWorkloadCluster(ctx context.Context, clusterKey client.ObjectKey) (WorkloadCluster, error) -} - -// Management holds operations on the management cluster. -type Management struct { - Client client.Reader - Tracker *remote.ClusterCacheTracker -} - -// RemoteClusterConnectionError represents a failure to connect to a remote cluster. -type RemoteClusterConnectionError struct { - Name string - Err error -} - -// Error satisfies the error interface. -func (e *RemoteClusterConnectionError) Error() string { return e.Name + ": " + e.Err.Error() } - -// Unwrap satisfies the unwrap error inteface. -func (e *RemoteClusterConnectionError) Unwrap() error { return e.Err } - -// Get implements ctrlclient.Reader -func (m *Management) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { - return m.Client.Get(ctx, key, obj) -} - -// List implements ctrlclient.Reader -func (m *Management) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { - return m.Client.List(ctx, list, opts...) -} - -// GetMachinesForCluster returns a list of machines that can be filtered or not. -// If no filter is supplied then all machines associated with the target cluster are returned. -func (m *Management) GetMachinesForCluster(ctx context.Context, cluster *clusterv1.Cluster, filters ...collections.Func) (collections.Machines, error) { - return collections.GetFilteredMachinesForCluster(ctx, m.Client, cluster, filters...) -} - -// GetMachinePoolsForCluster returns a list of machine pools owned by the cluster. -func (m *Management) GetMachinePoolsForCluster(ctx context.Context, cluster *clusterv1.Cluster) (*expv1.MachinePoolList, error) { - selectors := []client.ListOption{ - client.InNamespace(cluster.GetNamespace()), - client.MatchingLabels{ - clusterv1.ClusterLabelName: cluster.GetName(), - }, - } - machinePoolList := &expv1.MachinePoolList{} - err := m.Client.List(ctx, machinePoolList, selectors...) - return machinePoolList, err -} - -// GetWorkloadCluster builds a cluster object. -// The cluster comes with an etcd client generator to connect to any etcd pod living on a managed machine. -func (m *Management) GetWorkloadCluster(ctx context.Context, clusterKey client.ObjectKey) (WorkloadCluster, error) { - restConfig, err := remote.RESTConfig(ctx, K3ssControlPlaneControllerName, m.Client, clusterKey) - if err != nil { - return nil, err - } - restConfig.Timeout = 30 * time.Second - - if m.Tracker == nil { - return nil, errors.New("Cannot get WorkloadCluster: No remote Cluster Cache") - } - - c, err := m.Tracker.GetClient(ctx, clusterKey) - if err != nil { - return nil, err - } - - return &Workload{ - Client: c, - }, nil -} diff --git a/controlplane/k3s/pkg/cluster/cluster_labels.go b/controlplane/k3s/pkg/cluster/cluster_labels.go deleted file mode 100644 index 2a2bfc39..00000000 --- a/controlplane/k3s/pkg/cluster/cluster_labels.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cluster - -import ( - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" -) - -// ControlPlaneMachineLabelsForCluster returns a set of labels to add to a control plane machine for this specific cluster. -func ControlPlaneMachineLabelsForCluster(kcp *infracontrolplanev1.K3sControlPlane, clusterName string) map[string]string { - labels := map[string]string{} - - // Add the labels from the MachineTemplate. - // Note: we intentionally don't use the map directly to ensure we don't modify the map in KCP. - for k, v := range kcp.Spec.MachineTemplate.ObjectMeta.Labels { - labels[k] = v - } - - // Always force these labels over the ones coming from the spec. - labels[clusterv1.ClusterLabelName] = clusterName - labels[clusterv1.MachineControlPlaneLabelName] = "" - return labels -} diff --git a/controlplane/k3s/pkg/cluster/control_plane.go b/controlplane/k3s/pkg/cluster/control_plane.go deleted file mode 100644 index 33161b23..00000000 --- a/controlplane/k3s/pkg/cluster/control_plane.go +++ /dev/null @@ -1,290 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cluster - -import ( - "context" - - "github.com/go-logr/logr" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - kerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apiserver/pkg/storage/names" - "k8s.io/klog/v2/klogr" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/external" - controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" - "sigs.k8s.io/cluster-api/util/collections" - "sigs.k8s.io/cluster-api/util/failuredomains" - "sigs.k8s.io/cluster-api/util/patch" - "sigs.k8s.io/controller-runtime/pkg/client" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" -) - -// Log is the global logger for the internal package. -var Log = klogr.New() - -// ControlPlane holds business logic around control planes. -// It should never need to connect to a service, that responsibility lies outside of this struct. -// Going forward we should be trying to add more logic to here and reduce the amount of logic in the reconciler. -type ControlPlane struct { - KCP *infracontrolplanev1.K3sControlPlane - Cluster *clusterv1.Cluster - Machines collections.Machines - machinesPatchHelpers map[string]*patch.Helper - - // reconciliationTime is the time of the current reconciliation, and should be used for all "now" calculations - reconciliationTime metav1.Time - - // TODO: we should see if we can combine these with the Machine objects so we don't have all these separate lookups - // See discussion on https://github.com/kubernetes-sigs/cluster-api/pull/3405 - k3sConfigs map[string]*infrabootstrapv1.K3sConfig - infraResources map[string]*unstructured.Unstructured -} - -// NewControlPlane returns an instantiated ControlPlane. -func NewControlPlane(ctx context.Context, client client.Client, cluster *clusterv1.Cluster, kcp *infracontrolplanev1.K3sControlPlane, ownedMachines collections.Machines) (*ControlPlane, error) { - infraObjects, err := getInfraResources(ctx, client, ownedMachines) - if err != nil { - return nil, err - } - k3sConfigs, err := getK3sConfigs(ctx, client, ownedMachines) - if err != nil { - return nil, err - } - patchHelpers := map[string]*patch.Helper{} - for _, machine := range ownedMachines { - patchHelper, err := patch.NewHelper(machine, client) - if err != nil { - return nil, errors.Wrapf(err, "failed to create patch helper for machine %s", machine.Name) - } - patchHelpers[machine.Name] = patchHelper - } - - return &ControlPlane{ - KCP: kcp, - Cluster: cluster, - Machines: ownedMachines, - machinesPatchHelpers: patchHelpers, - k3sConfigs: k3sConfigs, - infraResources: infraObjects, - reconciliationTime: metav1.Now(), - }, nil -} - -// Logger returns a logger with useful context. -func (c *ControlPlane) Logger() logr.Logger { - return Log.WithValues("namespace", c.KCP.Namespace, "name", c.KCP.Name, "cluster-name", c.Cluster.Name) -} - -// FailureDomains returns a slice of failure domain objects synced from the infrastructure provider into Cluster.Status. -func (c *ControlPlane) FailureDomains() clusterv1.FailureDomains { - if c.Cluster.Status.FailureDomains == nil { - return clusterv1.FailureDomains{} - } - return c.Cluster.Status.FailureDomains -} - -// Version returns the K3sControlPlane's version. -func (c *ControlPlane) Version() *string { - return &c.KCP.Spec.Version -} - -// MachineInfrastructureTemplateRef returns the K3sControlPlane's infrastructure template for Machines. -func (c *ControlPlane) MachineInfrastructureTemplateRef() *corev1.ObjectReference { - return &c.KCP.Spec.MachineTemplate.InfrastructureRef -} - -// AsOwnerReference returns an owner reference to the K3sControlPlane. -func (c *ControlPlane) AsOwnerReference() *metav1.OwnerReference { - return &metav1.OwnerReference{ - APIVersion: infracontrolplanev1.GroupVersion.String(), - Kind: "K3sControlPlane", - Name: c.KCP.Name, - UID: c.KCP.UID, - } -} - -// MachineInFailureDomainWithMostMachines returns the first matching failure domain with machines that has the most control-plane machines on it. -func (c *ControlPlane) MachineInFailureDomainWithMostMachines(machines collections.Machines) (*clusterv1.Machine, error) { - fd := c.FailureDomainWithMostMachines(machines) - machinesInFailureDomain := machines.Filter(collections.InFailureDomains(fd)) - machineToMark := machinesInFailureDomain.Oldest() - if machineToMark == nil { - return nil, errors.New("failed to pick control plane Machine to mark for deletion") - } - return machineToMark, nil -} - -// MachineWithDeleteAnnotation returns a machine that has been annotated with DeleteMachineAnnotation key. -func (c *ControlPlane) MachineWithDeleteAnnotation(machines collections.Machines) collections.Machines { - // See if there are any machines with DeleteMachineAnnotation key. - annotatedMachines := machines.Filter(collections.HasAnnotationKey(clusterv1.DeleteMachineAnnotation)) - // If there are, return list of annotated machines. - return annotatedMachines -} - -// FailureDomainWithMostMachines returns a fd which exists both in machines and control-plane machines and has the most -// control-plane machines on it. -func (c *ControlPlane) FailureDomainWithMostMachines(machines collections.Machines) *string { - // See if there are any Machines that are not in currently defined failure domains first. - notInFailureDomains := machines.Filter( - collections.Not(collections.InFailureDomains(c.FailureDomains().FilterControlPlane().GetIDs()...)), - ) - if len(notInFailureDomains) > 0 { - // return the failure domain for the oldest Machine not in the current list of failure domains - // this could be either nil (no failure domain defined) or a failure domain that is no longer defined - // in the cluster status. - return notInFailureDomains.Oldest().Spec.FailureDomain - } - return failuredomains.PickMost(c.Cluster.Status.FailureDomains.FilterControlPlane(), c.Machines, machines) -} - -// NextFailureDomainForScaleUp returns the failure domain with the fewest number of up-to-date machines. -func (c *ControlPlane) NextFailureDomainForScaleUp() *string { - if len(c.Cluster.Status.FailureDomains.FilterControlPlane()) == 0 { - return nil - } - return failuredomains.PickFewest(c.FailureDomains().FilterControlPlane(), c.UpToDateMachines()) -} - -// InitialControlPlaneConfig returns a new K3sConfigSpec that is to be used for an initializing control plane. -func (c *ControlPlane) InitialControlPlaneConfig() *infrabootstrapv1.K3sConfigSpec { - bootstrapSpec := c.KCP.Spec.K3sConfigSpec.DeepCopy() - bootstrapSpec.AgentConfiguration = nil - return bootstrapSpec -} - -// JoinControlPlaneConfig returns a new K3sConfigSpec that is to be used for joining control planes. -func (c *ControlPlane) JoinControlPlaneConfig() *infrabootstrapv1.K3sConfigSpec { - bootstrapSpec := c.KCP.Spec.K3sConfigSpec.DeepCopy() - bootstrapSpec.AgentConfiguration = nil - return bootstrapSpec -} - -// GenerateK3sConfig generates a new k3s config for creating new control plane nodes. -func (c *ControlPlane) GenerateK3sConfig(spec *infrabootstrapv1.K3sConfigSpec) *infrabootstrapv1.K3sConfig { - // Create an owner reference without a controller reference because the owning controller is the machine controller - owner := metav1.OwnerReference{ - APIVersion: controlplanev1.GroupVersion.String(), - Kind: "K3sControlPlane", - Name: c.KCP.Name, - UID: c.KCP.UID, - } - - bootstrapConfig := &infrabootstrapv1.K3sConfig{ - ObjectMeta: metav1.ObjectMeta{ - Name: names.SimpleNameGenerator.GenerateName(c.KCP.Name + "-"), - Namespace: c.KCP.Namespace, - Labels: ControlPlaneMachineLabelsForCluster(c.KCP, c.Cluster.Name), - Annotations: c.KCP.Spec.MachineTemplate.ObjectMeta.Annotations, - OwnerReferences: []metav1.OwnerReference{owner}, - }, - Spec: *spec, - } - return bootstrapConfig -} - -// getInfraResources fetches the external infrastructure resource for each machine in the collection and returns a map of machine.Name -> infraResource. -func getInfraResources(ctx context.Context, cl client.Client, machines collections.Machines) (map[string]*unstructured.Unstructured, error) { - result := map[string]*unstructured.Unstructured{} - for _, m := range machines { - infraObj, err := external.Get(ctx, cl, &m.Spec.InfrastructureRef, m.Namespace) - if err != nil { - if apierrors.IsNotFound(errors.Cause(err)) { - continue - } - return nil, errors.Wrapf(err, "failed to retrieve infra obj for machine %q", m.Name) - } - result[m.Name] = infraObj - } - return result, nil -} - -// getK3sConfigs fetches the kubeadm config for each machine in the collection and returns a map of machine.Name -> K3sConfig. -func getK3sConfigs(ctx context.Context, cl client.Client, machines collections.Machines) (map[string]*infrabootstrapv1.K3sConfig, error) { - result := map[string]*infrabootstrapv1.K3sConfig{} - for _, m := range machines { - bootstrapRef := m.Spec.Bootstrap.ConfigRef - if bootstrapRef == nil { - continue - } - machineConfig := &infrabootstrapv1.K3sConfig{} - if err := cl.Get(ctx, client.ObjectKey{Name: bootstrapRef.Name, Namespace: m.Namespace}, machineConfig); err != nil { - if apierrors.IsNotFound(errors.Cause(err)) { - continue - } - return nil, errors.Wrapf(err, "failed to retrieve bootstrap config for machine %q", m.Name) - } - result[m.Name] = machineConfig - } - return result, nil -} - -// HasDeletingMachine returns true if any machine in the control plane is in the process of being deleted. -func (c *ControlPlane) HasDeletingMachine() bool { - return len(c.Machines.Filter(collections.HasDeletionTimestamp)) > 0 -} - -// MachinesNeedingRollout return a list of machines that need to be rolled out. -func (c *ControlPlane) MachinesNeedingRollout() collections.Machines { - // Ignore machines to be deleted. - machines := c.Machines.Filter(collections.Not(collections.HasDeletionTimestamp)) - - // Return machines if they are scheduled for rollout or if with an outdated configuration. - return machines.AnyFilter( - // Machines that are scheduled for rollout (KCP.Spec.RolloutAfter set, the RolloutAfter deadline is expired, and the machine was created before the deadline). - collections.ShouldRolloutAfter(&c.reconciliationTime, c.KCP.Spec.RolloutAfter), - // Machines that do not match with KCP config. - collections.Not(MatchesMachineSpec(c.infraResources, c.k3sConfigs, c.KCP)), - ) -} - -// UpToDateMachines returns the machines that are up to date with the control -// plane's configuration and therefore do not require rollout. -func (c *ControlPlane) UpToDateMachines() collections.Machines { - return c.Machines.Filter( - // Machines that shouldn't be rolled out after the deadline has expired. - collections.Not(collections.ShouldRolloutAfter(&c.reconciliationTime, c.KCP.Spec.RolloutAfter)), - // Machines that match with KCP config. - MatchesMachineSpec(c.infraResources, c.k3sConfigs, c.KCP), - ) -} - -// PatchMachines patches all the machines conditions. -func (c *ControlPlane) PatchMachines(ctx context.Context) error { - errList := make([]error, 0) - for i := range c.Machines { - machine := c.Machines[i] - if helper, ok := c.machinesPatchHelpers[machine.Name]; ok { - if err := helper.Patch(ctx, machine, patch.WithOwnedConditions{Conditions: []clusterv1.ConditionType{ - infracontrolplanev1.MachineAgentHealthyCondition, - infracontrolplanev1.MachineEtcdMemberHealthyCondition, - }}); err != nil { - errList = append(errList, errors.Wrapf(err, "failed to patch machine %s", machine.Name)) - } - continue - } - errList = append(errList, errors.Errorf("failed to get patch helper for machine %s", machine.Name)) - } - return kerrors.NewAggregate(errList) -} diff --git a/controlplane/k3s/pkg/cluster/doc.go b/controlplane/k3s/pkg/cluster/doc.go deleted file mode 100644 index 86c2fc56..00000000 --- a/controlplane/k3s/pkg/cluster/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package cluster contains internal implementation details for the k3s Control Plane. -package cluster diff --git a/controlplane/k3s/pkg/cluster/filters.go b/controlplane/k3s/pkg/cluster/filters.go deleted file mode 100644 index bd139893..00000000 --- a/controlplane/k3s/pkg/cluster/filters.go +++ /dev/null @@ -1,241 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cluster - -import ( - "encoding/json" - "reflect" - - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/util/collections" - "sigs.k8s.io/controller-runtime/pkg/client" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" -) - -// MatchesMachineSpec returns a filter to find all machines that matches with KCP config and do not require any rollout. -// Kubernetes version, infrastructure template, and K3sConfig field need to be equivalent. -func MatchesMachineSpec(infraConfigs map[string]*unstructured.Unstructured, machineConfigs map[string]*infrabootstrapv1.K3sConfig, kcp *infracontrolplanev1.K3sControlPlane) func(machine *clusterv1.Machine) bool { - return collections.And( - func(machine *clusterv1.Machine) bool { - return matchMachineTemplateMetadata(kcp, machine) - }, - collections.MatchesKubernetesVersion(kcp.Spec.Version), - //MatchesK3sBootstrapConfig(machineConfigs, kcp), - MatchesTemplateClonedFrom(infraConfigs, kcp), - ) -} - -// MatchesTemplateClonedFrom returns a filter to find all machines that match a given KCP infra template. -func MatchesTemplateClonedFrom(infraConfigs map[string]*unstructured.Unstructured, kcp *infracontrolplanev1.K3sControlPlane) collections.Func { - return func(machine *clusterv1.Machine) bool { - if machine == nil { - return false - } - infraObj, found := infraConfigs[machine.Name] - if !found { - // Return true here because failing to get infrastructure machine should not be considered as unmatching. - return true - } - - clonedFromName, ok1 := infraObj.GetAnnotations()[clusterv1.TemplateClonedFromNameAnnotation] - clonedFromGroupKind, ok2 := infraObj.GetAnnotations()[clusterv1.TemplateClonedFromGroupKindAnnotation] - if !ok1 || !ok2 { - // All kcp cloned infra machines should have this annotation. - // Missing the annotation may be due to older version machines or adopted machines. - // Should not be considered as mismatch. - return true - } - - // Check if the machine's infrastructure reference has been created from the current KCP infrastructure template. - if clonedFromName != kcp.Spec.MachineTemplate.InfrastructureRef.Name || - clonedFromGroupKind != kcp.Spec.MachineTemplate.InfrastructureRef.GroupVersionKind().GroupKind().String() { - return false - } - - // Check if the machine template metadata matches with the infrastructure object. - if !matchMachineTemplateMetadata(kcp, infraObj) { - return false - } - return true - } -} - -// MatchesK3sBootstrapConfig checks if machine's K3sConfigSpec is equivalent with KCP's K3sConfigSpec. -func MatchesK3sBootstrapConfig(machineConfigs map[string]*infrabootstrapv1.K3sConfig, kcp *infracontrolplanev1.K3sControlPlane) collections.Func { - return func(machine *clusterv1.Machine) bool { - if machine == nil { - return false - } - - // Check if KCP and machine ClusterConfiguration matches, if not return - if match := matchClusterConfiguration(kcp, machine); !match { - return false - } - - bootstrapRef := machine.Spec.Bootstrap.ConfigRef - if bootstrapRef == nil { - // Missing bootstrap reference should not be considered as unmatching. - // This is a safety precaution to avoid selecting machines that are broken, which in the future should be remediated separately. - return true - } - - machineConfig, found := machineConfigs[machine.Name] - if !found { - // Return true here because failing to get K3sConfig should not be considered as unmatching. - // This is a safety precaution to avoid rolling out machines if the client or the api-server is misbehaving. - return true - } - - // Check if the machine template metadata matches with the infrastructure object. - if !matchMachineTemplateMetadata(kcp, machineConfig) { - return false - } - - // Check if KCP and machine InitConfiguration or JoinConfiguration matches - // NOTE: only one between init configuration and join configuration is set on a machine, depending - // on the fact that the machine was the initial control plane node or a joining control plane node. - return matchInitOrJoinConfiguration(machineConfig, kcp) - } -} - -// matchClusterConfiguration verifies if KCP and machine ClusterConfiguration matches. -// NOTE: Machines that have K3sClusterConfigurationAnnotation will have to match with KCP ClusterConfiguration. -// If the annotation is not present (machine is either old or adopted), we won't roll out on any possible changes -// made in KCP's ClusterConfiguration given that we don't have enough information to make a decision. -// Users should use KCP.Spec.RolloutAfter field to force a rollout in this case. -func matchClusterConfiguration(kcp *infracontrolplanev1.K3sControlPlane, machine *clusterv1.Machine) bool { - machineClusterConfigStr, ok := machine.GetAnnotations()[infracontrolplanev1.K3sServerConfigurationAnnotation] - if !ok { - // We don't have enough information to make a decision; don't' trigger a roll out. - return true - } - - machineClusterConfig := &infrabootstrapv1.Cluster{} - // ClusterConfiguration annotation is not correct, only solution is to rollout. - // The call to json.Unmarshal has to take a pointer to the pointer struct defined above, - // otherwise we won't be able to handle a nil ClusterConfiguration (that is serialized into "null"). - // See https://github.com/kubernetes-sigs/cluster-api/issues/3353. - if err := json.Unmarshal([]byte(machineClusterConfigStr), &machineClusterConfig); err != nil { - return false - } - - // If any of the compared values are nil, treat them the same as an empty ClusterConfiguration. - if machineClusterConfig == nil { - machineClusterConfig = &infrabootstrapv1.Cluster{} - } - kcpLocalClusterConfiguration := kcp.Spec.K3sConfigSpec.Cluster - if kcpLocalClusterConfiguration == nil { - kcpLocalClusterConfiguration = &infrabootstrapv1.Cluster{} - } - - // Compare and return. - return reflect.DeepEqual(machineClusterConfig, kcpLocalClusterConfiguration) -} - -// matchInitOrJoinConfiguration verifies if KCP and machine ServerConfiguration or AgentConfiguration matches. -// NOTE: By extension this method takes care of detecting changes in other fields of the K3sConfig configuration (e.g. Files, Mounts etc.) -func matchInitOrJoinConfiguration(machineConfig *infrabootstrapv1.K3sConfig, kcp *infracontrolplanev1.K3sControlPlane) bool { - if machineConfig == nil { - // Return true here because failing to get K3sConfig should not be considered as unmatching. - // This is a safety precaution to avoid rolling out machines if the client or the api-server is misbehaving. - return true - } - - // takes the K3sConfigSpec from KCP and applies the transformations required - // to allow a comparison with the K3sConfig referenced from the machine. - kcpConfig := getAdjustedKcpConfig(kcp, machineConfig) - - // Default both K3sConfigSpecs before comparison. - // *Note* This assumes that newly added default values never - // introduce a semantic difference to the unset value. - // But that is something that is ensured by our API guarantees. - infrabootstrapv1.DefaultK3sConfigSpec(kcpConfig) - infrabootstrapv1.DefaultK3sConfigSpec(&machineConfig.Spec) - - // cleanups all the fields that are not relevant for the comparison. - cleanupConfigFields(kcpConfig, machineConfig) - - return reflect.DeepEqual(&machineConfig.Spec, kcpConfig) -} - -// getAdjustedKcpConfig takes the K3sConfigSpec from KCP and applies the transformations required -// to allow a comparison with the K3sConfig referenced from the machine. -// NOTE: The KCP controller applies a set of transformations when creating a K3sConfig referenced from the machine, -// mostly depending on the fact that the machine was the initial control plane node or a joining control plane node. -// In this function we don't have such information, so we are making the K3sConfigSpec similar to the KubeadmConfig. -func getAdjustedKcpConfig(kcp *infracontrolplanev1.K3sControlPlane, machineConfig *infrabootstrapv1.K3sConfig) *infrabootstrapv1.K3sConfigSpec { - kcpConfig := kcp.Spec.K3sConfigSpec.DeepCopy() - - // Machine's join configuration is nil when it is the first machine in the control plane. - if machineConfig.Spec.AgentConfiguration == nil { - kcpConfig.AgentConfiguration = nil - } - - // Machine's init configuration is nil when the control plane is already initialized. - if machineConfig.Spec.ServerConfiguration == nil { - kcpConfig.ServerConfiguration = nil - } - - return kcpConfig -} - -// cleanupConfigFields cleanups all the fields that are not relevant for the comparison. -func cleanupConfigFields(kcpConfig *infrabootstrapv1.K3sConfigSpec, machineConfig *infrabootstrapv1.K3sConfig) { - // KCP ClusterConfiguration will only be compared with a machine's ClusterConfiguration annotation, so - // we are cleaning up from the reflect.DeepEqual comparison. - kcpConfig.Cluster = nil - machineConfig.Spec.Cluster = nil - - kcpConfig.ServerConfiguration = nil - machineConfig.Spec.ServerConfiguration = nil - - // If KCP JoinConfiguration is not present, set machine JoinConfiguration to nil (nothing can trigger rollout here). - // NOTE: this is required because CABPK applies an empty joinConfiguration in case no one is provided. - if kcpConfig.AgentConfiguration == nil { - machineConfig.Spec.AgentConfiguration = nil - } -} - -// matchMachineTemplateMetadata matches the machine template object meta information, -// specifically annotations and labels, against an object. -func matchMachineTemplateMetadata(kcp *infracontrolplanev1.K3sControlPlane, obj client.Object) bool { - // Check if annotations and labels match. - if !isSubsetMapOf(kcp.Spec.MachineTemplate.ObjectMeta.Annotations, obj.GetAnnotations()) { - return false - } - if !isSubsetMapOf(kcp.Spec.MachineTemplate.ObjectMeta.Labels, obj.GetLabels()) { - return false - } - return true -} - -func isSubsetMapOf(base map[string]string, existing map[string]string) bool { -loopBase: - for key, value := range base { - for existingKey, existingValue := range existing { - if existingKey == key && existingValue == value { - continue loopBase - } - } - // Return false right away if a key value pair wasn't found. - return false - } - return true -} diff --git a/controlplane/k3s/pkg/cluster/workload_cluster.go b/controlplane/k3s/pkg/cluster/workload_cluster.go deleted file mode 100644 index e735f1a6..00000000 --- a/controlplane/k3s/pkg/cluster/workload_cluster.go +++ /dev/null @@ -1,101 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cluster - -import ( - "context" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/cluster-api/util" - ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - labelNodeRoleControlPlane = "node-role.kubernetes.io/master" -) - -// WorkloadCluster defines all behaviors necessary to upgrade kubernetes on a workload cluster -// -// TODO: Add a detailed description to each of these method definitions. -type WorkloadCluster interface { - // Basic health and status checks. - ClusterStatus(ctx context.Context) (Status, error) - UpdateAgentConditions(ctx context.Context, controlPlane *ControlPlane) - UpdateEtcdConditions(ctx context.Context, controlPlane *ControlPlane) -} - -// Workload defines operations on workload clusters. -type Workload struct { - Client ctrlclient.Client -} - -// Status holds stats information about the cluster. -type Status struct { - // Nodes are a total count of nodes - Nodes int32 - // ReadyNodes are the count of nodes that are reporting ready - ReadyNodes int32 -} - -func (w *Workload) getControlPlaneNodes(ctx context.Context) (*corev1.NodeList, error) { - controlPlaneNodes := &corev1.NodeList{} - controlPlaneNodeNames := sets.NewString() - - nodes := &corev1.NodeList{} - labels := map[string]string{ - labelNodeRoleControlPlane: "true", - } - if err := w.Client.List(ctx, nodes, ctrlclient.MatchingLabels(labels)); err != nil { - return nil, err - } - - for i := range nodes.Items { - node := nodes.Items[i] - - // Continue if we already added that node. - if controlPlaneNodeNames.Has(node.Name) { - continue - } - - controlPlaneNodeNames.Insert(node.Name) - controlPlaneNodes.Items = append(controlPlaneNodes.Items, node) - } - - return controlPlaneNodes, nil -} - -// ClusterStatus returns the status of the cluster. -func (w *Workload) ClusterStatus(ctx context.Context) (Status, error) { - status := Status{} - - // count the control plane nodes - nodes, err := w.getControlPlaneNodes(ctx) - if err != nil { - return status, err - } - - for _, node := range nodes.Items { - nodeCopy := node - status.Nodes++ - if util.IsNodeReady(&nodeCopy) { - status.ReadyNodes++ - } - } - - return status, nil -} diff --git a/controlplane/k3s/pkg/cluster/workload_cluster_conditions.go b/controlplane/k3s/pkg/cluster/workload_cluster_conditions.go deleted file mode 100644 index 30c3146e..00000000 --- a/controlplane/k3s/pkg/cluster/workload_cluster_conditions.go +++ /dev/null @@ -1,305 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cluster - -import ( - "context" - "fmt" - "strings" - - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" - "sigs.k8s.io/cluster-api/util/collections" - "sigs.k8s.io/cluster-api/util/conditions" - ctrl "sigs.k8s.io/controller-runtime" - ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" - - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" -) - -// UpdateEtcdConditions is responsible for updating machine conditions reflecting the status of all the etcd members. -// This operation is best effort, in the sense that in case of problems in retrieving member status, it sets -// the condition to Unknown state without returning any error. -func (w *Workload) UpdateEtcdConditions(ctx context.Context, controlPlane *ControlPlane) { - w.updateManagedEtcdConditions(ctx, controlPlane) -} - -func (w *Workload) updateManagedEtcdConditions(ctx context.Context, controlPlane *ControlPlane) { - // NOTE: This methods uses control plane nodes only to get in contact with etcd but then it relies on etcd - // as ultimate source of truth for the list of members and for their health. - controlPlaneNodes, err := w.getControlPlaneNodes(ctx) - if err != nil { - conditions.MarkUnknown(controlPlane.KCP, infracontrolplanev1.EtcdClusterHealthyCondition, infracontrolplanev1.EtcdClusterInspectionFailedReason, "Failed to list nodes which are hosting the etcd members") - for _, m := range controlPlane.Machines { - conditions.MarkUnknown(m, infracontrolplanev1.MachineEtcdMemberHealthyCondition, infracontrolplanev1.EtcdMemberInspectionFailedReason, "Failed to get the node which is hosting the etcd member") - } - return - } - - for _, node := range controlPlaneNodes.Items { - var machine *clusterv1.Machine - for _, m := range controlPlane.Machines { - if m.Status.NodeRef != nil && m.Status.NodeRef.Name == node.Name { - machine = m - } - } - - if machine == nil { - // If there are machines still provisioning there is the chance that a chance that a node might be linked to a machine soon, - // otherwise report the error at KCP level given that there is no machine to report on. - if hasProvisioningMachine(controlPlane.Machines) { - continue - } - continue - } - - // If the machine is deleting, report all the conditions as deleting - if !machine.ObjectMeta.DeletionTimestamp.IsZero() { - conditions.MarkFalse(machine, infracontrolplanev1.MachineEtcdMemberHealthyCondition, clusterv1.DeletingReason, clusterv1.ConditionSeverityInfo, "") - continue - } - - conditions.MarkTrue(machine, infracontrolplanev1.MachineEtcdMemberHealthyCondition) - } -} - -// UpdateAgentConditions is responsible for updating machine conditions reflecting the status of all the control plane -// components running in a static pod generated by kubeadm. This operation is best effort, in the sense that in case -// of problems in retrieving the pod status, it sets the condition to Unknown state without returning any error. -func (w *Workload) UpdateAgentConditions(ctx context.Context, controlPlane *ControlPlane) { - log := ctrl.LoggerFrom(ctx) - log.Info("Updating Agent conditions") - - allMachinePodConditions := []clusterv1.ConditionType{ - infracontrolplanev1.MachineAgentHealthyCondition, - } - - // NOTE: this fun uses control plane nodes from the workload cluster as a source of truth for the current state. - controlPlaneNodes, err := w.getControlPlaneNodes(ctx) - if err != nil { - for i := range controlPlane.Machines { - machine := controlPlane.Machines[i] - for _, condition := range allMachinePodConditions { - conditions.MarkUnknown(machine, condition, infracontrolplanev1.PodInspectionFailedReason, "Failed to get the node which is hosting this component") - } - } - conditions.MarkUnknown(controlPlane.KCP, infracontrolplanev1.ControlPlaneComponentsHealthyCondition, infracontrolplanev1.ControlPlaneComponentsInspectionFailedReason, "Failed to list nodes which are hosting control plane components") - return - } - - // Update conditions for control plane components hosted as static pods on the nodes. - var kcpErrors []string - - for _, node := range controlPlaneNodes.Items { - // Search for the machine corresponding to the node. - var machine *clusterv1.Machine - for _, m := range controlPlane.Machines { - if m.Status.NodeRef != nil && m.Status.NodeRef.Name == node.Name { - machine = m - break - } - } - - // If there is no machine corresponding to a node, determine if this is an error or not. - if machine == nil { - // If there are machines still provisioning there is the chance that a chance that a node might be linked to a machine soon, - // otherwise report the error at KCP level given that there is no machine to report on. - if hasProvisioningMachine(controlPlane.Machines) { - continue - } - kcpErrors = append(kcpErrors, fmt.Sprintf("Control plane node %s does not have a corresponding machine", node.Name)) - continue - } - - // If the machine is deleting, report all the conditions as deleting - if !machine.ObjectMeta.DeletionTimestamp.IsZero() { - for _, condition := range allMachinePodConditions { - conditions.MarkFalse(machine, condition, clusterv1.DeletingReason, clusterv1.ConditionSeverityInfo, "") - } - continue - } - - // If the node is Unreachable, information about static pods could be stale so set all conditions to unknown. - if nodeHasUnreachableTaint(node) { - // NOTE: We are assuming unreachable as a temporary condition, leaving to MHC - // the responsibility to determine if the node is unhealthy or not. - for _, condition := range allMachinePodConditions { - conditions.MarkUnknown(machine, condition, infracontrolplanev1.PodInspectionFailedReason, "Node is unreachable") - } - continue - } - - targetnode := corev1.Node{} - nodeKey := ctrlclient.ObjectKey{ - Namespace: metav1.NamespaceSystem, - Name: node.Name, - } - - if err := w.Client.Get(ctx, nodeKey, &targetnode); err != nil { - // If there is an error getting the Pod, do not set any conditions. - if apierrors.IsNotFound(err) { - conditions.MarkFalse(machine, infracontrolplanev1.MachineAgentHealthyCondition, infracontrolplanev1.PodMissingReason, clusterv1.ConditionSeverityError, "Node %s is missing", nodeKey.Name) - - return - } - conditions.MarkUnknown(machine, infracontrolplanev1.MachineAgentHealthyCondition, infracontrolplanev1.PodInspectionFailedReason, "Failed to get node status") - return - } - - for _, condition := range targetnode.Status.Conditions { - if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { - conditions.MarkTrue(machine, infracontrolplanev1.MachineAgentHealthyCondition) - } - } - } - - // If there are provisioned machines without corresponding nodes, report this as a failing conditions with SeverityError. - for i := range controlPlane.Machines { - machine := controlPlane.Machines[i] - if machine.Status.NodeRef == nil { - continue - } - found := false - for _, node := range controlPlaneNodes.Items { - if machine.Status.NodeRef.Name == node.Name { - found = true - break - } - } - if !found { - for _, condition := range allMachinePodConditions { - conditions.MarkFalse(machine, condition, controlplanev1.PodFailedReason, clusterv1.ConditionSeverityError, "Missing node") - } - } - } - - // Aggregate components error from machines at KCP level. - aggregateFromMachinesToKCP(aggregateFromMachinesToKCPInput{ - controlPlane: controlPlane, - machineConditions: allMachinePodConditions, - kcpErrors: kcpErrors, - condition: controlplanev1.ControlPlaneComponentsHealthyCondition, - unhealthyReason: controlplanev1.ControlPlaneComponentsUnhealthyReason, - unknownReason: controlplanev1.ControlPlaneComponentsUnknownReason, - note: "control plane", - }) -} - -func hasProvisioningMachine(machines collections.Machines) bool { - for _, machine := range machines { - if machine.Status.NodeRef == nil { - return true - } - } - return false -} - -// nodeHasUnreachableTaint returns true if the node has is unreachable from the node controller. -func nodeHasUnreachableTaint(node corev1.Node) bool { - for _, taint := range node.Spec.Taints { - if taint.Key == corev1.TaintNodeUnreachable && taint.Effect == corev1.TaintEffectNoExecute { - return true - } - } - return false -} - -type aggregateFromMachinesToKCPInput struct { - controlPlane *ControlPlane - machineConditions []clusterv1.ConditionType - kcpErrors []string - condition clusterv1.ConditionType - unhealthyReason string - unknownReason string - note string -} - -// aggregateFromMachinesToKCP aggregates a group of conditions from machines to KCP. -// NOTE: this func follows the same aggregation rules used by conditions.Merge thus giving priority to -// errors, then warning, info down to unknown. -func aggregateFromMachinesToKCP(input aggregateFromMachinesToKCPInput) { - // Aggregates machines for condition status. - // NB. A machine could be assigned to many groups, but only the group with the highest severity will be reported. - kcpMachinesWithErrors := sets.NewString() - kcpMachinesWithWarnings := sets.NewString() - kcpMachinesWithInfo := sets.NewString() - kcpMachinesWithTrue := sets.NewString() - kcpMachinesWithUnknown := sets.NewString() - - for i := range input.controlPlane.Machines { - machine := input.controlPlane.Machines[i] - for _, condition := range input.machineConditions { - if machineCondition := conditions.Get(machine, condition); machineCondition != nil { - switch machineCondition.Status { - case corev1.ConditionTrue: - kcpMachinesWithTrue.Insert(machine.Name) - case corev1.ConditionFalse: - switch machineCondition.Severity { - case clusterv1.ConditionSeverityInfo: - kcpMachinesWithInfo.Insert(machine.Name) - case clusterv1.ConditionSeverityWarning: - kcpMachinesWithWarnings.Insert(machine.Name) - case clusterv1.ConditionSeverityError: - kcpMachinesWithErrors.Insert(machine.Name) - } - case corev1.ConditionUnknown: - kcpMachinesWithUnknown.Insert(machine.Name) - } - } - } - } - - // In case of at least one machine with errors or KCP level errors (nodes without machines), report false, error. - if len(kcpMachinesWithErrors) > 0 { - input.kcpErrors = append(input.kcpErrors, fmt.Sprintf("Following machines are reporting %s errors: %s", input.note, strings.Join(kcpMachinesWithErrors.List(), ", "))) - } - if len(input.kcpErrors) > 0 { - conditions.MarkFalse(input.controlPlane.KCP, input.condition, input.unhealthyReason, clusterv1.ConditionSeverityError, strings.Join(input.kcpErrors, "; ")) - return - } - - // In case of no errors and at least one machine with warnings, report false, warnings. - if len(kcpMachinesWithWarnings) > 0 { - conditions.MarkFalse(input.controlPlane.KCP, input.condition, input.unhealthyReason, clusterv1.ConditionSeverityWarning, "Following machines are reporting %s warnings: %s", input.note, strings.Join(kcpMachinesWithWarnings.List(), ", ")) - return - } - - // In case of no errors, no warning, and at least one machine with info, report false, info. - if len(kcpMachinesWithWarnings) > 0 { - conditions.MarkFalse(input.controlPlane.KCP, input.condition, input.unhealthyReason, clusterv1.ConditionSeverityWarning, "Following machines are reporting %s info: %s", input.note, strings.Join(kcpMachinesWithInfo.List(), ", ")) - return - } - - // In case of no errors, no warning, no Info, and at least one machine with true conditions, report true. - if len(kcpMachinesWithTrue) > 0 { - conditions.MarkTrue(input.controlPlane.KCP, input.condition) - return - } - - // Otherwise, if there is at least one machine with unknown, report unknown. - if len(kcpMachinesWithUnknown) > 0 { - conditions.MarkUnknown(input.controlPlane.KCP, input.condition, input.unknownReason, "Following machines are reporting unknown %s status: %s", input.note, strings.Join(kcpMachinesWithUnknown.List(), ", ")) - return - } - - // This last case should happen only if there are no provisioned machines, and thus without conditions. - // So there will be no condition at KCP level too. -} diff --git a/controlplane/k3s/pkg/kubeconfig/kubeconfig.go b/controlplane/k3s/pkg/kubeconfig/kubeconfig.go deleted file mode 100644 index 023a9470..00000000 --- a/controlplane/k3s/pkg/kubeconfig/kubeconfig.go +++ /dev/null @@ -1,189 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package kubeconfig implements the kubeconfig generation logic. -package kubeconfig - -import ( - "context" - "crypto" - "crypto/x509" - "fmt" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/clientcmd/api" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/cluster-api/util/certs" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/kubesphere/kubekey/v3/util/secret" -) - -var ( - // ErrDependentCertificateNotFound is returned when the dependent certificate is not found. - ErrDependentCertificateNotFound = errors.New("could not find secret ca") -) - -// New creates a new Kubeconfig using the cluster name and specified endpoint. -func New(clusterName, endpoint string, clientCACert *x509.Certificate, clientCAKey crypto.Signer, serverCACert *x509.Certificate) (*api.Config, error) { - cfg := &certs.Config{ - CommonName: "kubernetes-admin", - Organization: []string{"system:masters"}, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - } - - clientKey, err := certs.NewPrivateKey() - if err != nil { - return nil, errors.Wrap(err, "unable to create private key") - } - - clientCert, err := cfg.NewSignedCert(clientKey, clientCACert, clientCAKey) - if err != nil { - return nil, errors.Wrap(err, "unable to sign certificate") - } - - userName := fmt.Sprintf("%s-admin", clusterName) - contextName := fmt.Sprintf("%s@%s", userName, clusterName) - - return &api.Config{ - Clusters: map[string]*api.Cluster{ - clusterName: { - Server: endpoint, - CertificateAuthorityData: certs.EncodeCertPEM(serverCACert), - }, - }, - Contexts: map[string]*api.Context{ - contextName: { - Cluster: clusterName, - AuthInfo: userName, - }, - }, - AuthInfos: map[string]*api.AuthInfo{ - userName: { - ClientKeyData: certs.EncodePrivateKeyPEM(clientKey), - ClientCertificateData: certs.EncodeCertPEM(clientCert), - }, - }, - CurrentContext: contextName, - }, nil -} - -// CreateSecret creates the Kubeconfig secret for the given cluster. -func CreateSecret(ctx context.Context, c client.Client, cluster *clusterv1.Cluster) error { - name := util.ObjectKey(cluster) - return CreateSecretWithOwner(ctx, c, name, cluster.Spec.ControlPlaneEndpoint.String(), metav1.OwnerReference{ - APIVersion: clusterv1.GroupVersion.String(), - Kind: "Cluster", - Name: cluster.Name, - UID: cluster.UID, - }) -} - -// CreateSecretWithOwner creates the Kubeconfig secret for the given cluster name, namespace, endpoint, and owner reference. -func CreateSecretWithOwner(ctx context.Context, c client.Client, clusterName client.ObjectKey, endpoint string, owner metav1.OwnerReference) error { - server := fmt.Sprintf("https://%s", endpoint) - out, err := generateKubeconfig(ctx, c, clusterName, server) - if err != nil { - return err - } - - return c.Create(ctx, GenerateSecretWithOwner(clusterName, out, owner)) -} - -// GenerateSecret returns a Kubernetes secret for the given Cluster and kubeconfig data. -func GenerateSecret(cluster *clusterv1.Cluster, data []byte) *corev1.Secret { - name := util.ObjectKey(cluster) - return GenerateSecretWithOwner(name, data, metav1.OwnerReference{ - APIVersion: clusterv1.GroupVersion.String(), - Kind: "Cluster", - Name: cluster.Name, - UID: cluster.UID, - }) -} - -// GenerateSecretWithOwner returns a Kubernetes secret for the given Cluster name, namespace, kubeconfig data, and ownerReference. -func GenerateSecretWithOwner(clusterName client.ObjectKey, data []byte, owner metav1.OwnerReference) *corev1.Secret { - return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secret.Name(clusterName.Name, secret.Kubeconfig), - Namespace: clusterName.Namespace, - Labels: map[string]string{ - clusterv1.ClusterLabelName: clusterName.Name, - }, - OwnerReferences: []metav1.OwnerReference{ - owner, - }, - }, - Data: map[string][]byte{ - secret.KubeconfigDataName: data, - }, - } -} - -func generateKubeconfig(ctx context.Context, c client.Client, clusterName client.ObjectKey, endpoint string) ([]byte, error) { - clusterCA, err := secret.GetFromNamespacedName(ctx, c, clusterName, secret.ClusterCA) - if err != nil { - if apierrors.IsNotFound(errors.Cause(err)) { - return nil, ErrDependentCertificateNotFound - } - return nil, err - } - - clientClusterCA, err := secret.GetFromNamespacedName(ctx, c, clusterName, secret.ClientClusterCA) - if err != nil { - if apierrors.IsNotFound(errors.Cause(err)) { - return nil, ErrDependentCertificateNotFound - } - return nil, err - } - - clientCACert, err := certs.DecodeCertPEM(clientClusterCA.Data[secret.TLSCrtDataName]) - if err != nil { - return nil, errors.Wrap(err, "failed to decode CA Cert") - } else if clientCACert == nil { - return nil, errors.New("certificate not found in config") - } - - clientCAKey, err := certs.DecodePrivateKeyPEM(clientClusterCA.Data[secret.TLSKeyDataName]) - if err != nil { - return nil, errors.Wrap(err, "failed to decode private key") - } else if clientCAKey == nil { - return nil, errors.New("CA private key not found") - } - - serverCACert, err := certs.DecodeCertPEM(clusterCA.Data[secret.TLSCrtDataName]) - if err != nil { - return nil, errors.Wrap(err, "failed to decode CA Cert") - } else if serverCACert == nil { - return nil, errors.New("certificate not found in config") - } - - cfg, err := New(clusterName.Name, endpoint, clientCACert, clientCAKey, serverCACert) - if err != nil { - return nil, errors.Wrap(err, "failed to generate a kubeconfig") - } - - out, err := clientcmd.Write(*cfg) - if err != nil { - return nil, errors.Wrap(err, "failed to serialize config to yaml") - } - return out, nil -} diff --git a/docs/CAPKK-Developer-Guide.md b/docs/CAPKK-Developer-Guide.md deleted file mode 100644 index 67619948..00000000 --- a/docs/CAPKK-Developer-Guide.md +++ /dev/null @@ -1,169 +0,0 @@ -# CAPKK Developer Guide - -CAPKK (Cluster-API-Provider-Kubekey) is an SSH-based infrastructure provider that follows the development documentation of the cluster-api Provider contracts. (https://cluster-api.sigs.k8s.io/developer/providers/contracts.html#provider-contract) - -In the CAPKK project, there are three main resources and corresponding controllers: KKCluster, KKMachine, and KKInstance. - -## Project structure - -```bash -├── api -│ └── v1beta1 ## CR defined in CAPKK -├── bootstrap ## bootstrap provider for k3s version -├── config ## Kubernetes yaml files for CAPKK project -├── controllers ## controllers included in CAPKK -│ ├── kkcluster -│ ├── kkinstance -│ └── kkmachine -├── controlplane ## control-plane provider for k3s version -├── docs ## documentation -├── exp ## experimental features -├── hack -├── pkg -│ ├── clients ## clients used in CAPKK -│ │ └── ssh -│ ├── rootfs ## data storage directory for CAPKK -│ ├── scope ## interfaces defined by functionality in CAPKK, mainly used to split KKCluster, KKMachine, and KKInstance attribute fields. -│ ├── service ## concrete implementation of various operations in CAPKK -│ │ ├── binary ## operations on binary files in CAPKK -│ │ ├── bootstrap ## operations on machine environment initialization in CAPKK -│ │ ├── containermanager ## operations on container runtime in CAPKK -│ │ ├── operation ## basic operations on Linux machines in CAPKK -│ │ ├── provisioning ## parsing cloud-init or ignition files in CAPKK and mapping them to SSH commands -│ │ ├── repository ## operations on rpm packages in CAPKK -│ │ └── util -│ └── util -│ ├── filesystem -│ └── hash -├── scripts -├── templates ## used to generate automated scripts in sample yaml for GitHub releases -├── test ## e2e testing in CAPKK -│ └── e2e -├── util ## general utility class for KubeKey -└── version ## contains KubeKey's version and SHA256 of binary -```` - -## KKCluster -KKCluster is used to define the global basic information of a cluster. -```go -type KKClusterSpec struct { - // Define the distribution of Kubernetes, with currently supported parameters including "kubernetes" and "k3s". - Distribution string `json:"distribution,omitempty"` - - // Define the SSH authentication information for the nodes. - Nodes Nodes `json:"nodes"` - - // Define the endpoints for the control plane. - // Optional - ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` - - // Define the address for the load balancer for the control plane. - ControlPlaneLoadBalancer *KKLoadBalancerSpec `json:"controlPlaneLoadBalancer,omitempty"` - - // Define the download address for the cluster component binaries, which is usually used to overwrite default addresses in offline deployment scenarios. - // Optional - Component *Component `json:"component,omitempty"` - - // Define information for the private image repository. - // Optional - Registry Registry `json:"registry,omitempty"` -} -``` -In the above content, the following points should be noted: -* The nodes field specifies the SSH information of all nodes included in the cluster. -* CAPKK uses kube-vip by default to achieve high availability of the cluster control plane, so the controlPlaneLoadBalancer can be set to any unused IP within the subnet. -* CAPKK uses the official download address of binary components by default. If you want to use the domestic resource address maintained by the KubeSphere team, you can configure it as follows: -```yaml -spec: - component: - zone: cn -``` -* The component field can also be used to specify a custom file server address and configure the overrides array to modify the path and SHA256 checksum of different binaries in the file server used by CAPKK. - -The following is a common KKCluster sample YAML file: -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKCluster -metadata: - name: quick-start -spec: - component: - zone: "" - nodes: - auth: - user: ubuntu - password: P@88w0rd - instances: - - name: test1 - address: 192.168.0.3 - - name: test2 - address: 192.168.0.4 - - name: test3 - address: 192.168.0.5 - controlPlaneLoadBalancer: - host: 192.168.0.100 -``` -After applying this YAML file, the webhook of KKCluster will add some default values, such as `roles: [control-plane, worker]`. -## KKMachine -KKMachine is mainly used to represent the mapping between Kubernetes node and infrastructure host. -```go -type KKMachineSpec struct { - // Unique ID of the infrastructure provider - ProviderID *string `json:"providerID,omitempty"` - - // Name of the corresponding KKInstance resource - InstanceID *string `json:"instanceID,omitempty"` - - // Role of the node - // Optional - Roles []Role `json:"roles"` - - // Container runtime of the corresponding node for this resource - // Optional - ContainerManager ContainerManager `json:"containerManager"` - - // software packages of the corresponding node for this resource - // Optional - Repository *Repository `json:"repository,omitempty"` -} -``` -Usually, we do not directly use KKMachine resources, but declare KKMachineTemplate resources for reference by cluster-api resources. - - -### KKMachineTemplate -The xxxTemplate resource is the template type of this type of resource, which contains all the contents in the spec of the corresponding resource. In the usage of cluster-api, we declare resources with the replica field, such as KubeadmControlPlane (KCP) and MachineDeployment (MD), indicating that this type of resource can be dynamically scaled. These resources also contain an infrastructureRef field, which refers to a specific infrastructure template resource (xxxMachineTemplate). When scaling up, KCP and MD resources will create corresponding, real single instances of the resource based on the template resource. -The following is an example of a common control plane KKMachineTemplate yaml: -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKMachineTemplate -metadata: - name: quick-start - namespace: default -spec: - template: - spec: - roles: - - control-plane - containerManager: - type: containerd - repository: - iso: "none" - update: false -``` -Notes: -- The roles specify control-plane, which means that the resources that reference this KKMachineTemplate will only select machines with corresponding roles in KKCluster for bootstrapping. -- containerManager specifies that the container runtime configured in the KKMachine created by this template is containerd. -- repository specifies the policy for the KKMachine to operate rpm software sources created by this template. In this case, the policy indicates that CAPKK will use the default software source on the corresponding Linux machine to install default software dependencies (conntrack, socat, etc.). -## KKInstance -In the developer documentation of cluster-api, the definition of infra only includes xxxCluster and xxxMachine resources, while KKInstance is a resource exclusive to CAPKK. The additional definition of KKInstance in CAPKK aims to decouple the logic of the operator and controller, i.e., KKMachine is focused on maintaining interactions with cluster-api CR, while KKInstance focuses on maintaining Linux machines (by performing command-based operations on the machines via SSH). -The property fields of KKInstance mainly consist of the collection of nodes and KKMachine fields in KKCluster, which will not be elaborated here. - -The controller of KKInstance mainly performs the following operations on the machine: - -* Bootstrop: Some machine environment initialization or cleanup operations defined in CAPKK, such as creating corresponding directories, adding Linux users, running initialization shell scripts, resetting the network, deleting files, and resetting Kubeadm. -* Repository: RPM software source handling operations defined in CAPKK, such as mounting ISO software packages, updating sources, and installing dependency software packages. -* Binary: Cluster component binary file handling operations defined in CAPKK, such as downloading binary files. -* ContainerManager: Machine container runtime operations defined in CAPKK, such as checking container runtime and installing container runtime. -* Provisioning: Parsing cloud-init or ignition files provided by cluster-api for the corresponding machine in CAPKK and mapping them to SSH commands. This cloud-init file will include operations such as "kubeadm init" and "kubeadm join". - -For the interface definitions of these operations, see [interface](https://github.com/kubesphere/kubekey/blob/master/pkg/service/interface.go). diff --git a/docs/addons.md b/docs/addons.md deleted file mode 100644 index bdc16bae..00000000 --- a/docs/addons.md +++ /dev/null @@ -1,87 +0,0 @@ -Addons ------------- - -All plugins which are installed by yaml or chart can be kubernetes' addons. So the addons configuration support both yaml and chart. - -Explanation of parameters: -```yaml -- name: xxx # the name of addon - namespace: xxx # namespace - sources: # support both yaml and chart - chart: - name: xxx # the name of chart - repo: xxx # the name of chart repo (url) - path: xxx # the location of chart (path) - values: xxx # specify values for chart (string list) - valuesFile: xxx # specify values file for chart (path / url) - yaml: - path: [] # the location list of yaml (path / url) -``` -example: -```yaml -apiVersion: kubekey.kubesphere.io/v1alpha2 -kind: Cluster -metadata: - name: example -spec: - hosts: - - {name: node1, address: 172.16.0.2, internalAddress: 172.16.0.2, privateKeyPath: "~/.ssh/id_rsa"} - - {name: node2, address: 172.16.0.3, internalAddress: 172.16.0.3, privateKeyPath: "~/.ssh/id_rsa"} - - {name: node3, address: 172.16.0.4, internalAddress: 172.16.0.4, privateKeyPath: "~/.ssh/id_rsa"} - ... - addons: - - name: nfs-client - namespace: kube-system - sources: - chart: - name: nfs-client-provisioner - repo: https://charts.kubesphere.io/main - valuesFile: /mycluster/nfs/custom-nfs-client-values.yaml # or https://raw.githubusercontent.com/kubesphere/helm-charts/master/src/main/nfs-client-provisioner/values.yaml - # values also supports parameter lists - # values: - # - storageClass.defaultClass=true - # - nfs.server=192.168.6.3 - # - nfs.path=/mnt/kubesphere - - - name: glusterfs - namespace: kube-system - sources: - yaml: - path: - - /mycluster/glusterfs/glusterfs.yaml # or https://raw.githubusercontent.com/xxx/glusterfs.yaml - - - name: sonarqube - namespace: test - sources: - chart: - name: sonarqube - repo: https://charts.kubesphere.io/main - - - name: csi-qingcloud - namespace: kube-system - sources: - chart: - name: csi-qingcloud - repo: https://charts.kubesphere.io/test - values: - - config.qy_access_key_id=*** - - config.qy_secret_access_key=*** - - config.zone=*** - - sc.isDefaultClass=true - - - name: rbd-provisioner - namespace: kube-system - sources: - chart: - name: rbd-provisioner - repo: https://charts.kubesphere.io/test - values: - # for more values, see https://github.com/kubesphere/helm-charts/tree/master/src/test/rbd-provisioner - - ceph.mon=*** - - ceph.pool=*** - - ceph.adminId=*** - - ceph.adminKey=*** - - ceph.userId=*** - - ceph.userKey=*** - - sc.isDefault=true -``` diff --git a/docs/check-renew-certificate.md b/docs/check-renew-certificate.md deleted file mode 100644 index 56ecbc47..00000000 --- a/docs/check-renew-certificate.md +++ /dev/null @@ -1,45 +0,0 @@ -### Certificate -#### Check certificate expiration -```shell script -./kk certs check-expiration [(-f | --file) path] - --f to specify the configuration file which was generated for cluster creation. This parameter is not required if it is single node. - -./kk certs check-expiration -INFO[21:33:59 CST] Listing cluster certs ... -CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY NODE -apiserver.crt Dec 18, 2021 08:27 UTC 352d ca node1 -apiserver-kubelet-client.crt Dec 18, 2021 08:27 UTC 352d ca node1 -front-proxy-client.crt Dec 18, 2021 08:27 UTC 352d front-proxy-ca node1 -admin.conf Dec 18, 2021 08:27 UTC 352d node1 -controller-manager.conf Dec 18, 2021 08:27 UTC 352d node1 -scheduler.conf Dec 18, 2021 08:27 UTC 352d node1 - -CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME NODE -ca.crt Dec 16, 2030 08:27 UTC 9y node1 -front-proxy-ca.crt Dec 16, 2030 08:27 UTC 9y node1 -INFO[21:34:00 CST] Successful. -``` - -#### Renew certificate -```shell script -./kk certs renew [(-f | --file) path] - --f to specify the configuration file which was generated for cluster creation. This parameter is not required if it is single node. - -./kk certs renew -INFO[21:42:51 CST] Renewing cluster certs ... -INFO[21:42:54 CST] Syncing cluster kubeConfig ... -INFO[21:42:55 CST] Listing cluster certs ... -CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY NODE -apiserver.crt Dec 30, 2021 13:42 UTC 364d ca node1 -apiserver-kubelet-client.crt Dec 30, 2021 13:42 UTC 364d ca node1 -front-proxy-client.crt Dec 30, 2021 13:42 UTC 364d front-proxy-ca node1 -admin.conf Dec 30, 2021 13:42 UTC 364d node1 -controller-manager.conf Dec 30, 2021 13:42 UTC 364d node1 -scheduler.conf Dec 30, 2021 13:42 UTC 364d node1 - -CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME NODE -ca.crt Dec 16, 2030 08:27 UTC 9y node1 -front-proxy-ca.crt Dec 16, 2030 08:27 UTC 9y node1 -``` diff --git a/docs/cluster-autoscaler.md b/docs/cluster-autoscaler.md deleted file mode 100644 index 098a5d56..00000000 --- a/docs/cluster-autoscaler.md +++ /dev/null @@ -1,172 +0,0 @@ -# Cluster-Autoscaler for capkk -refer to https://cluster-api.sigs.k8s.io/tasks/automated-machine-management/autoscaling.html -capkk is the infrastructure cluster for clusterapi.Here is an example of deploying Cluster Autoscaler in capkk. - -## Deployment plan - - - -1. Install the autoscaler on the master cluster to manage the dynamic scaling of the workload cluster. -2. Install the autoscaler and the cluster it manages in the same namespace. - -## Scaling mechanism - - - -⚠️ **Automatic scaling is only supported for worker nodes.** - -1. Configure all scalable machines on the nodes of the kkcluster. - -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKCluster -metadata: - name: capkk-1 - namespace: default -spec: - component: - zone: cn - controlPlaneLoadBalancer: - host: 172.31.53.163 - nodes: - auth: - password: "123456" - user: root - instances: - - address: 172.31.53.163 - - address: 172.31.53.160 - - address: 172.31.53.122 -``` - -2. Configure annotations on the machinedeployment to allow the autoscaler to discover node groups. - -```yaml -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - annotations: - cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "3" - cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "0" - capacity.cluster-autoscaler.kubernetes.io/memory: "16G" - capacity.cluster-autoscaler.kubernetes.io/cpu: "8" - name: capkk-1-md-0 - namespace: default -spec: - clusterName: capkk-1 - replicas: 1 - selector: - matchLabels: null - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: capkk-1-md-0 - clusterName: capkk-1 - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: KKMachineTemplate - name: capkk-1-md-0 - version: v1.25.4 -``` - -When `cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size` is set to "0", it indicates that the minimum number of worker nodes is 0. In this case, it is necessary to set `capacity.cluster-autoscaler.kubernetes.io/memory` and `capacity.cluster-autoscaler.kubernetes.io/cpu`. - -3. Modify the startup parameters in the autoscaler deployment - -```yaml ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cluster-autoscaler - labels: - app: cluster-autoscaler -spec: - selector: - matchLabels: - app: cluster-autoscaler - replicas: 1 - template: - metadata: - labels: - app: cluster-autoscaler - spec: - containers: - - image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.26.3 - name: default - command: - - /cluster-autoscaler - args: - - --cloud-provider=clusterapi - - --kubeconfig=/tmp/kubeconfig/workload.conf - - --clusterapi-cloud-config-authoritative - - --node-group-auto-discovery=clusterapi:namespace=${NAMESPACE} - - --scale-down-enabled=false - volumeMounts: - - mountPath: /tmp/kubeconfig - name: workload-kubeconfig - serviceAccountName: cluster-autoscaler - terminationGracePeriodSeconds: 10 - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - volumes: - - name: workload-kubeconfig - secret: - secretName: '${CLUSTER_NAME}-kubeconfig' - optional: true - items: - - key: value - path: workload.conf ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cluster-autoscaler-management -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: cluster-autoscaler-management -subjects: - - kind: ServiceAccount - name: cluster-autoscaler - namespace: '${NAMESPACE}' ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cluster-autoscaler ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cluster-autoscaler-management -rules: - - apiGroups: - - cluster.x-k8s.io - resources: - - machinedeployments - - machinedeployments/scale - - machines - - machinesets - verbs: - - get - - list - - update - - watch - - apiGroups: - - infrastructure.cluster.x-k8s.io - resources: - - kkmachinetemplates - verbs: - - get - - list - - update - - watch -``` - -**ScalingUp**：Triggered when there are no nodes available for scheduling pods. - -**ScalingDown**: Configure it in the startup parameters of the autoscaler. For the complete configuration, please refer to the official website of the autoscaler. diff --git a/docs/commands/kk-add-nodes.md b/docs/commands/kk-add-nodes.md deleted file mode 100644 index 0e630d60..00000000 --- a/docs/commands/kk-add-nodes.md +++ /dev/null @@ -1,47 +0,0 @@ -# NAME -**kk add nodes**: Add nodes to the cluster according to the new nodes information from the specified configuration file. - -# DESCRIPTION -Add nodes to the cluster according to the new nodes information from the specified configuration file. You need to add new node's information to the cluster config file first, then apply the changes. - -# OPTIONS - -## **--filename, -f** -Path to a configuration file. - -## **--skip-pull-images** -Skip pre pull images. The default is `false`. - -## **--container-manager** -Container manager: docker, crio, containerd and isula. The default is `docker`. - -## **--download-cmd** -The user defined command to download the necessary binary files. The first param `%s` is output path, the second param `%s`, is the URL. The default is `curl -L -o %s %s`. - -## **--artifact, -a** -Path to a KubeKey artifact. - -## **--with-packages** -Install operating system packages by artifact. The default is `false`. - -## **--in-cluster** -Running inside the cluster. The default is `false`. - -## **--debug** -Print detailed information. The default is `false`. - -## **--yes, -y** -Skip confirm check. The default is `false`. - -## **--ignore-err** -Ignore the error message, remove the host which reported error and force to continue. The default is `false`. - -# EXAMPLES -Add nodes from the specified configuration file. -``` -$ kk add nodes -f config-sample.yaml -``` -Add nodes from the specified configuration file and use the artifact to install operating system packages. -``` -$ kk add nodes -f config-sample.yaml -a kubekey-artifact.tar.gz --with-packages -``` \ No newline at end of file diff --git a/docs/commands/kk-add.md b/docs/commands/kk-add.md deleted file mode 100644 index d70b1d7b..00000000 --- a/docs/commands/kk-add.md +++ /dev/null @@ -1,10 +0,0 @@ -# NAME -**kk add**: Add nodes to kubernetes cluster. - -# DESCRIPTION -Add nodes to kubernetes cluster. - -# COMMANDS -| Command | Description | -| - | - | -| [kk add nodes](./kk-add-nodes.md) | Add nodes to the cluster according to the new nodes information from the specified configuration file. | diff --git a/docs/commands/kk-artifact-export.md b/docs/commands/kk-artifact-export.md deleted file mode 100644 index c3122c44..00000000 --- a/docs/commands/kk-artifact-export.md +++ /dev/null @@ -1,25 +0,0 @@ -# NAME -**kk artifact export**: Export a KubeKey offline installation package. - -# DESCRIPTION -**kk** will base on the specified manifest file to pull all images, download the specified binaries and Linux repository iso file, then archive them as a KubeKey offline installation package. The export command will download the corresponding binaries from the Internet, so please make sure the network connection is success. - -# OPTIONS - -## **--manifest, -m** -Path to a manifest file. This option is required. - -## **--output, -o** -Path to a output path The default is `kubekey-artifact.tar.gz`. - -## **--download-cmd** -The user defined command to download the necessary binary files. The first param `%s` is output path, the second param `%s`, is the URL. The default is `curl -L -o %s %s`. - -## **--debug** -Print detailed information. The default is `false`. - -# EXAMPLES -Export a KubeKey artifact named `my-artifact.tar.gz`. -``` -$ kk artifact export -m manifest-sample.yaml -o my-artifact.tar.gz -``` \ No newline at end of file diff --git a/docs/commands/kk-artifact-images-push.md b/docs/commands/kk-artifact-images-push.md deleted file mode 100644 index 0d8792fb..00000000 --- a/docs/commands/kk-artifact-images-push.md +++ /dev/null @@ -1,29 +0,0 @@ -# NAME -**kk artifact images push**: Push images to a registry from a KubeKey artifact. - -# DESCRIPTION -Push images to a registry from a KubeKey artifact. - -# OPTIONS - -## **--filename, -f** -Path to a configuration file. - -## **--images-dir** -Path to a KubeKey artifact images directory (e.g. ./kubekey/images). - -## **--artifact, -a** -Path to a KubeKey artifact. - -## **--debug** -Print detailed information. The default is `false`. - -# EXAMPLES -Push the image to the private image registry. -``` -$ kk artifact images push -f config-sample.yaml -a kubekey-artifact.tar.gz -``` -Push the image to the private image registry from a specify directory. -``` -$ kk artifact images push -f config-sample.yaml --images-dir ./kubekey/images -``` \ No newline at end of file diff --git a/docs/commands/kk-artifact-images.md b/docs/commands/kk-artifact-images.md deleted file mode 100644 index da8a7a45..00000000 --- a/docs/commands/kk-artifact-images.md +++ /dev/null @@ -1,15 +0,0 @@ -# NAME -**kk artifact images**: Manage KubeKey artifact image. - -# ALIASES -**kk artifact image** *command* - -**kk artifact i** *command* - -# DESCRIPTION -Manage KubeKey artifact image. - -# COMMANDS -| Command | Description | -| - | - | -| [kk artifact images push](./kk-artifact-images-push.md) | Push images to a registry from a KubeKey artifact. | \ No newline at end of file diff --git a/docs/commands/kk-artifact-import.md b/docs/commands/kk-artifact-import.md deleted file mode 100644 index 91905ccd..00000000 --- a/docs/commands/kk-artifact-import.md +++ /dev/null @@ -1,23 +0,0 @@ -# NAME -**kk artifact import**: Import a KubeKey offline installation package. - -# DESCRIPTION -The import command will unarchive the KubeKey offline installation package to get all images, specified binaries and Linux repository iso file. - -# OPTIONS - -## **--artifact, -a** -Path to a artifact gzip. This option is required. - -## **--with-packages** -Install operation system packages by artifact - -# EXAMPLES -import a KubeKey artifact named `my-artifact.tar.gz`. -``` -$ kk artifact import -a my-artifact.tar.gz -``` -import a KubeKey artifact named `my-artifact.tar.gz` and install local repository. -``` -$ kk artifact import -a my-artifact.tar.gz --with-packages true -``` \ No newline at end of file diff --git a/docs/commands/kk-artifact.md b/docs/commands/kk-artifact.md deleted file mode 100644 index 0d980f88..00000000 --- a/docs/commands/kk-artifact.md +++ /dev/null @@ -1,11 +0,0 @@ -# NAME -**kk artifact**: Manage a KubeKey offline installation package. - -# DESCRIPTION -Manage a KubeKey offline installation package. More information about the KubeKey artifact format can be found [here](../manifest_and_artifact.md). - -# COMMANDS -| Command | Description | -| - | - | -| [kk artifact export](./kk-artifact-export.md) | Export a KubeKey offline installation package. | -| [kk artifact images](./kk-artifact-images.md) | Manage KubeKey artifact images | \ No newline at end of file diff --git a/docs/commands/kk-certs-check-expiration.md b/docs/commands/kk-certs-check-expiration.md deleted file mode 100644 index 24c67def..00000000 --- a/docs/commands/kk-certs-check-expiration.md +++ /dev/null @@ -1,17 +0,0 @@ -# NAME -**kk certs check-expiration**: Check certificates expiration for a Kubernetes cluster. - -# DESCRIPTION -Check certificates expiration for a Kubernetes cluster. - -# OPTIONS - -## **--filename, -f** -Path to a configuration file. This option is required. - -# EXAMPLES -``` -$ kk certs check-expirtation -f config-example.yaml -``` - - diff --git a/docs/commands/kk-certs-renew.md b/docs/commands/kk-certs-renew.md deleted file mode 100644 index 7d8ebbd8..00000000 --- a/docs/commands/kk-certs-renew.md +++ /dev/null @@ -1,17 +0,0 @@ -# NAME -**kk certs renew**: Renew a cluster certs - -# DESCRIPTION -Renew a cluster certs. - -# OPTIONS - -## **--filename, -f** -Path to a configuration file. This option is required. - -# EXAMPLES -``` -$ kk certs renew -f config-example.yaml -``` - - diff --git a/docs/commands/kk-certs.md b/docs/commands/kk-certs.md deleted file mode 100644 index 992f6fc7..00000000 --- a/docs/commands/kk-certs.md +++ /dev/null @@ -1,11 +0,0 @@ -# NAME -**kk certs**:Manage cluster certs - -# DESCRIPTION -Manage cluster certs. - -# COMMANDS -| Command | Description | -| - | - | -| [kk certs check-expiration](./kk-certs-check-expiration.md) | Check certificates expiration for a Kubernetes cluster. | -| [kk certs renew](./kk-certs-renew.md) | Renew a cluster certs. | \ No newline at end of file diff --git a/docs/commands/kk-completion.md b/docs/commands/kk-completion.md deleted file mode 100644 index 3981edd8..00000000 --- a/docs/commands/kk-completion.md +++ /dev/null @@ -1,45 +0,0 @@ -# NAME -**kk completion**: Generate shell completion scripts. - -# DESCRIPTION -Generate shell completion scripts. Normally you don't need to do more extra work to have this feature if you've installed kk by brew. - -# OPTIONS - -## **--type** -Generate different types of shell. - -# EXAMPLES -Installing bash completion on Linux. -If bash-completion is not installed on Linux, please install the 'bash-completion' package via your distribution's package manager. -Load the ks completion code for bash into the current shell. -``` -source <(ks completion bash) -``` -Write bash completion code to a file and source if from .bash_profile. -``` -mkdir -p ~/.config/kk/ && kk completion --type bash > ~/.config/kk/completion.bash.inc -printf " -``` -kk shell completion. -``` -source '$HOME/.config/kk/completion.bash.inc' -" >> $HOME/.bash_profile -source $HOME/.bash_profile -``` - -In order to have good experience on zsh completion, ohmyzsh is a good choice. -Please install ohmyzsh by the following command. -``` -sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" -``` -Get more details about onmyzsh from: https://github.com/ohmyzsh/ohmyzsh - -Load the kk completion code for zsh[1] into the current shell. -``` -source <(kk completion --type zsh) -``` -Set the kk completion code for zsh[1] to autoload on startup. -``` -kk completion --type zsh > "${fpath[1]}/_kk" -``` \ No newline at end of file diff --git a/docs/commands/kk-create-cluster.md b/docs/commands/kk-create-cluster.md deleted file mode 100644 index 2682b35d..00000000 --- a/docs/commands/kk-create-cluster.md +++ /dev/null @@ -1,74 +0,0 @@ -# NAME -**kk create cluster**: Add nodes to the cluster according to the new nodes information from the specified configuration file. - -# DESCRIPTION -Add nodes to the cluster according to the new nodes information from the specified configuration file. You need to add new node's information to the cluster config file first, then apply the changes. - -# OPTIONS - -## **--artifact, -a** -Path to a KubeKey artifact. - -## **--certificates-dir** -Specifies where to store or look for all required certificates. - -## **--container-manager** -Container manager: docker, crio, containerd and isula. The default is `docker`. - -## **--debug** -Print detailed information. The default is `false`. - -## **--download-cmd** -The user defined command to download the necessary binary files. The first param `%s` is output path, the second param `%s`, is the URL. The default is `curl -L -o %s %s`. - -## **--filename, -f** -Path to a configuration file. - -## **--ignore-err** -Ignore the error message, remove the host which reported error and force to continue. The default is `false`. - -## **--in-cluster** -Running inside the cluster. The default is `false`. - -## **--skip-pull-images** -Skip pre pull images. The default is `false`. - -## **--skip-push-images** -Skip pre push images. The default is `false`. - -## **--with-kubernetes** -Specify a supported version of kubernetes. It will override the version of kubernetes in the config file. - -## **--with-kubesphere** -Deploy a specific version of kubesphere. It will override the kubesphere `ClusterConfiguration` in the config file with the default value. - -## **--with-local-storage** -Deploy a local PV provisioner. - -## **--with-packages** -Install operating system packages by artifact. The default is `false`. - -## **--yes, -y** -Skip confirm check. The default is `false`. - -# EXAMPLES -Create an `all-in-one` pure Kubernetes cluster with default version. -``` -$ kk create cluster -``` -Create a Kubernetes and KubeSphere cluster with a specified version. -``` -$ kk create cluster --with-kubernetes v1.21.5 --with-kubesphere v3.2.1 -``` -Create a cluster using the configuration file. -``` -$ kk create cluster -f config-sample.yaml -``` -Create a cluster from the specified configuration file and use the artifact to install operating system packages. -``` -$ kk create cluster -f config-sample.yaml -a kubekey-artifact.tar.gz --with-packages -``` -Create a cluster with the specified download command. -``` -$ kk create cluster --download-cmd 'hd get -t 8 -o %s %s' -``` \ No newline at end of file diff --git a/docs/commands/kk-create-config.md b/docs/commands/kk-create-config.md deleted file mode 100644 index b1183517..00000000 --- a/docs/commands/kk-create-config.md +++ /dev/null @@ -1,42 +0,0 @@ -# NAME -**kk create config**: Create cluster configuration file - -# DESCRIPTION -Create cluster configuration file. More information about the configuration file can be found in the [config-example.yaml](../config-example.md). - -# OPTIONS - -## **--debug** -Print detailed information. The default is `false`. - -## **--filename, -f** -Specify the configuration file output path. The default is `./config-example.yaml`. - -## **--from-cluster** -Create a configuration based on existing cluster. Usually used in conjunction with the ``--kubeconfig`` option. - -## **--kubeconfig** -Specify a kubeconfig file. - -## **--name** -Specify a name of cluster object. The default is `sample`. - -## **--with-kubernetes** -Specify a supported version of kubernetes. - -## **--with-kubesphere** -Deploy a specific version of kubesphere. It will generate the kubesphere `ClusterConfiguration` in the config file with the default value. - -# EXAMPLES -Create an example configuration file with default Kubernetes version. -``` -$ kk create config -``` -Create a Kubernetes and KubeSphere example configuration file with a specified version. -``` -$ kk create config --with-kubernetes v1.21.5 --with-kubesphere v3.2.1 -``` -Create a example configuration file based on existing cluster -``` -$ kk create config --from-cluster --kubeconfig ~/.kube/config -``` diff --git a/docs/commands/kk-create-manifest.md b/docs/commands/kk-create-manifest.md deleted file mode 100644 index 34bf24ac..00000000 --- a/docs/commands/kk-create-manifest.md +++ /dev/null @@ -1,30 +0,0 @@ -# NAME -**kk create manifest**: Create an offline installation package configuration file. - -# DESCRIPTION -Create an offline installation package configuration file. This command requires preparing a cluster environment that has been installed a Kubernetes cluster and providing the `kube config` file of the cluster for **kk**. More information about the KubeKey manifest file can be found in the [KubeKey Manifest and Artifact](../manifest_and_artifact.md) and [manifest-example.yaml](../manifest-example.md). - -# OPTIONS - -## **--debug** -Print detailed information. The default is `false`. - -## **--filename, -f** -Specify the manifest file output path. The default is `./manifest-example.yaml`. - -## **--kubeconfig** -Specify a kubeconfig file. The default is `$HOME/.kube/config`. - -## **--name** -Specify a name of manifest object. The default is `sample`. - -# EXAMPLES -Create an example manifest file based on the default `kube config ($HOME/.kube/config)` path. -``` -$ kk create manifest -``` -Create a manifest file with a specified `kube config` path. -``` -$ kk create manifest --kubeconfig /root/.kube/config -``` - diff --git a/docs/commands/kk-create-phase.md b/docs/commands/kk-create-phase.md deleted file mode 100644 index 32c618b2..00000000 --- a/docs/commands/kk-create-phase.md +++ /dev/null @@ -1,198 +0,0 @@ -# NAME -**kk alpha create phase**: Create your cluster in phases to a newer version with this command, which is not enough and need to add phase cmds. - -# DESCRIPTION -Create your cluster in phases to a newer version with this command, which is not enough and need to add phase cmds. - -# PHASES CMDS - -| Command | Description | -| - | - | -| kk alpha create phase binary | Download the binaries on the local. | -| kk alpha create phase os | Init the os configure. | -| kk alpha create phase images | Down the container and pull the images before creating your cluster. | -| kk alpha create phase etcd | Install the etcd on the master. | -| kk alpha create phase join | Join the control-plane nodes and worker nodes in the k8s cluster. | -| kk alpha create phase configure | Configure the k8s cluster with plugins, certs and PV. | -| kk alpha create phase kubesphere | Install the kubesphere with the input version. | - -# OPTIONS - -## **--artifact, -a** -Path to a KubeKey artifact. - -## **--certificates-dir** -Specifies where to store or look for all required certificates. - -## **--container-manager** -Container manager: docker, crio, containerd and isula. The default is `docker`. - -## **--debug** -Print detailed information. The default is `false`. - -## **--download-cmd** -The user defined command to download the necessary binary files. The first param `%s` is output path, the second param `%s`, is the URL. The default is `curl -L -o %s %s`. - -## **--filename, -f** -Path to a configuration file. - -## **--ignore-err** -Ignore the error message, remove the host which reported error and force to continue. The default is `false`. - -## **--in-cluster** -Running inside the cluster. The default is `false`. - -## **--skip-pull-images** -Skip pre pull images. The default is `false`. - -## **--skip-push-images** -Skip pre push images. The default is `false`. - -## **--with-kubernetes** -Specify a supported version of kubernetes. It will override the version of kubernetes in the config file. - -## **--with-kubesphere** -Deploy a specific version of kubesphere. It will override the kubesphere `ClusterConfiguration` in the config file with the default value. - -## **--with-local-storage** -Deploy a local PV provisioner. - -## **--with-packages** -Install operating system packages by artifact. The default is `false`. - -## **--yes, -y** -Skip confirm check. The default is `false`. - -# EXAMPLES -## Create an `all-in-one` pure Kubernetes cluster with default version -Download the binarys of creating an `all-in-one` Kubernetes cluster from a default version. -``` -$ kk alpha create phase binary -``` -Init the configure os of creating an `all-in-one` Kubernetes cluster from a default version. -``` -$ kk alpha create phase os -``` -Pull the images of creating an `all-in-one` Kubernetes cluster from a default version. -``` -$ kk alpha create phase images -``` -Install the etcd of creating an `all-in-one` Kubernetes cluster from a default version. -``` -$ kk alpha create phase etcd -``` -Init the k8s cluster of creating an `all-in-one` Kubernetes cluster from a default version. -``` -$ kk alpha create phase init -``` -Join the nodes to the k8s cluster of creating an `all-in-one` Kubernetes cluster from a default version. -``` -$ kk alpha create phase join -``` -Configure the k8s cluster of creating an `all-in-one` Kubernetes cluster from a default version. -``` -$ kk alpha create phase configure -``` -## Create an `all-in-one` pure Kubernetes cluster with specified version -Download the binarys of creating an `all-in-one` Kubernetes cluster from a specified version. -``` -$ kk alpha create phase binary --with-kubernetes v1.22.0 -``` -Init the configure os of creating an `all-in-one` Kubernetes cluster from a specified version. -``` -$ kk alpha create phase os -``` -Pull the images of creating an `all-in-one` Kubernetes cluster from a specified version. -``` -$ kk alpha create phase images --with-kubernetes v1.22.0 -``` -Install the etcd of creating an `all-in-one` Kubernetes cluster from a specified version. -``` -$ kk alpha create phase etcd -``` -Init the k8s cluster of creating an `all-in-one` Kubernetes cluster from a specified version. -``` -$ kk alpha create phase init --with-kubernetes v1.22.0 -``` -Join the nodes to the k8s cluster of creating an `all-in-one` Kubernetes cluster from a specified version. -``` -$ kk alpha create phase join --with-kubernetes v1.22.0 -``` -Configure the k8s cluster of creating an `all-in-one` Kubernetes cluster from a specified version. -``` -$ kk alpha create phase configure --with-kubernetes v1.22.0 -``` -Install the k8s cluster of creating an `all-in-one` Kubernetes cluster from a specified version. -``` -$ kk alpha create phase kubesphere --with-kubesphere v3.3.0 -``` -## Create a kubernetes cluster from a specified configuration file -Download the binarys of creating a kubernetes cluster from a specified configuration file. -``` -$ kk alpha create phase binary -f config-sample.yaml -``` -Init the configure os of creating a kubernetes cluster from a specified configuration file. -``` -$ kk alpha create phase os -``` -Pull the images of creating a kubernetes cluster from a specified configuration file. -``` -$ kk alpha create phase images -f config-sample.yaml -``` -Install the etcd of creating a kubernetes cluster from a specified configuration file. -``` -$ kk alpha create phase etcd -f config-sample.yaml -``` -Init the k8s cluster of creating a kubernetes cluster from a specified configuration file. -``` -$ kk alpha create phase init -f config-sample.yaml -``` -Join the nodes to the k8s cluster of creating a kubernetes cluster from a specified configuration file. -``` -$ kk alpha create phase join -f config-sample.yaml -``` -Configure the k8s cluster of creating a kubernetes cluster from a specified configuration file. -``` -$ kk alpha create phase configure -f config-sample.yaml -``` -Install the k8s cluster of creating a kubernetes cluster from a specified configuration file. -``` -$ kk alpha create phase kubesphere -f config-sample.yaml -``` -## Create a cluster from the specified configuration file and use the artifact to install operating system packages. -import a KubeKey artifact named `my-artifact.tar.gz`. -``` -$ kk artifact import -a my-artifact.tar.gz --with-packages -``` -Download the binarys of creating a cluster from the specified configuration file and use the artifact to install operating system packages. -``` -$ kk alpha create phase binary -f config-sample.yaml -``` -Init the configure os of creating a cluster from the specified configuration file and use the artifact to install operating system packages. -``` -$ kk alpha create phase os -``` -Pull the images of creating a cluster from the specified configuration file and use the artifact to install operating system packages. -``` -$ kk alpha create phase images -f config-sample.yaml -``` -Install the etcd of creating a cluster from the specified configuration file and use the artifact to install operating system packages. -``` -$ kk alpha create phase etcd -f config-sample.yaml -``` -Init the k8s cluster of creating a cluster from the specified configuration file and use the artifact to install operating system packages. -``` -$ kk alpha create phase init -f config-sample.yaml -``` -Join the nodes to the k8s cluster of creating a cluster from the specified configuration file and use the artifact to install operating system packages. -``` -$ kk alpha create phase join -f config-sample.yaml -``` -Configure the k8s cluster of creating a cluster from the specified configuration file and use the artifact to install operating system packages. -``` -$ kk alpha create phase configure -f config-sample.yaml -``` -Install the k8s cluster of creating a cluster from the specified configuration file and use the artifact to install operating system packages. -``` -$ kk alpha create phase kubesphere -f config-sample.yaml -``` \ No newline at end of file diff --git a/docs/commands/kk-create.md b/docs/commands/kk-create.md deleted file mode 100644 index 309d40e6..00000000 --- a/docs/commands/kk-create.md +++ /dev/null @@ -1,12 +0,0 @@ -# NAME -**kk create**: Create a cluster, a cluster configuration file or an offline installation package configuration file. - -# DESCRIPTION -Create a cluster, a cluster configuration file or an offline installation package configuration file. - -# COMMANDS -| Command | Description | -| - | - | -| [kk create cluster](./kk-create-cluster.md) | Create a Kubernetes or KubeSphere cluster. | -| [kk create config](./kk-create-config.md) | Create cluster configuration file. | -| [kk create manifest](./kk-create-manifest.md) | Create an offline installation package configuration file. | \ No newline at end of file diff --git a/docs/commands/kk-cri-migrate.md b/docs/commands/kk-cri-migrate.md deleted file mode 100644 index aed7fa4f..00000000 --- a/docs/commands/kk-cri-migrate.md +++ /dev/null @@ -1,48 +0,0 @@ -# NAME -**kk cri migrate**: migrate your cri smoothly to docker/containerd with this command. - -# DESCRIPTION -migrate your cri smoothly to docker/containerd with this command. - -# OPTIONS - -## **--role** -Which node(worker/master/all) to migrate. - -## **--type** -Which cri(docker/containerd) to migrate. - -## **--debug** -Print detailed information. The default is `false`. - -## **--filename, -f** -Path to a configuration file.This option is required. - -## **--yes, -y** -Skip confirm check. The default is `false`. - -# EXAMPLES -Migrate your master node's cri smoothly to docker. -``` -$ ./kk cri migrate --role master --type docker -f config-sample.yaml -``` -Migrate your master node's cri smoothly to containerd. -``` -$ ./kk cri migrate --role master --type containerd -f config-sample.yaml -``` -Migrate your worker node's cri smoothly to docker. -``` -$ ./kk cri migrate --role worker --type docker -f config-sample.yaml -``` -Migrate your worker node's cri smoothly to containerd. -``` -$ ./kk cri migrate --role worker --type containerd -f config-sample.yaml -``` -Migrate all your node's cri smoothly to containerd. -``` -$ ./kk cri migrate --role all --type containerd -f config-sample.yaml -``` -Migrate all your node's cri smoothly to docker. -``` -$ ./kk cri migrate --role all --type docker -f config-sample.yaml -``` \ No newline at end of file diff --git a/docs/commands/kk-delete-cluster.md b/docs/commands/kk-delete-cluster.md deleted file mode 100644 index 6370ba3e..00000000 --- a/docs/commands/kk-delete-cluster.md +++ /dev/null @@ -1,32 +0,0 @@ -# NAME -**kk delete cluster**: Delete a cluster. - -# DESCRIPTION -Delete a cluster. This command will use the `kubeadm reset` to reset all the nodes. Then, reset network policy, stop `etcd`, remove cluster directory, uninstall Kubernetes certs-auto-renew script and remove internal Loadbalancer module. And [network configurations](../network-configurations.md) on each node will be cleaned up. - -# OPTIONS - -## **--debug** -Print detailed information. The default is `false`. - -## **--filename, -f** -Path to a configuration file. - -## **--all, -A** -Delete all CRI(docker/containerd) related files and directories. - -# EXAMPLES -Delete an `all-in-one` cluster. -``` -$ kk delete cluster -``` -Delete a cluster from a specified configuration file. -``` -$ kk delete cluster -f config-example.yaml -``` -Delete a cluster included CRI related files and directories from a specified configuration file. -``` -$ kk delete cluster -f config-example.yaml --all -$ kk delete cluster -f config-example.yaml -A -``` - diff --git a/docs/commands/kk-delete-node.md b/docs/commands/kk-delete-node.md deleted file mode 100644 index d0835c0e..00000000 --- a/docs/commands/kk-delete-node.md +++ /dev/null @@ -1,21 +0,0 @@ -# NAME -**kk delete node**: Delete a node. - -# DESCRIPTION -Delete and cleanup a node. This command will use the `kubectl drain` to safely evict all pods, then use `kubectl delete node` to delete the specified node. And [network configurations](../network-configurations.md) on the node will be cleaned up. - -# OPTIONS - -## **--debug** -Print detailed information. The default is `false`. - -## **--filename, -f** -Path to a configuration file. - -# EXAMPLES -Delete a node named `node2` from a specified configuration file. -``` -$ kk delete node node2 -f config-example.yaml -``` - - diff --git a/docs/commands/kk-delete.md b/docs/commands/kk-delete.md deleted file mode 100644 index ea4759f6..00000000 --- a/docs/commands/kk-delete.md +++ /dev/null @@ -1,11 +0,0 @@ -# NAME -**kk delete**: Delete node or cluster. - -# DESCRIPTION -Delete node or cluster. - -# COMMANDS -| Command | Description | -| - | - | -| [kk delete cluster](./kk-delete-cluster.md) | Delete a cluster. | -| [kk delete node](./kk-delete-node.md) | Delete a node. | \ No newline at end of file diff --git a/docs/commands/kk-init-os.md b/docs/commands/kk-init-os.md deleted file mode 100644 index ee048ca8..00000000 --- a/docs/commands/kk-init-os.md +++ /dev/null @@ -1,27 +0,0 @@ -# NAME -**kk init os**: Init operating system. - -# DESCRIPTION -Init operating system. This command will install `openssl`, `socat`, `conntrack`, `ipset`, `ipvsadm`, `ebtables` and `chrony` on all the nodes. - -# OPTIONS - -## **--artifact, -a** -Path to a KubeKey artifact. - -## **--debug** -Print detailed information. The default is `false`. - -## **--filename, -f** -Path to a configuration file. - -# EXAMPLES -Init the operating system from a specified configuration file. -``` -$ kk init os -f config-example.yaml -``` -Init the operating system from a specified configuration file and use a KubeKey artifact. -``` -$ kk init os -f config-example.yaml -a kubekey-artifact.tar.gz -``` - diff --git a/docs/commands/kk-init-registry.md b/docs/commands/kk-init-registry.md deleted file mode 100644 index c4f4b254..00000000 --- a/docs/commands/kk-init-registry.md +++ /dev/null @@ -1,30 +0,0 @@ -# NAME -**kk init registry**: Init a local image registry. - -# DESCRIPTION -Init a local image registry. More information about the registry can be found [here](../registry.md). - -# OPTIONS - -## **--artifact, -a** -Path to a KubeKey artifact. - -## **--debug** -Print detailed information. The default is `false`. - -## **--download-cmd** -The user defined command to download the necessary binary files. The first param `%s` is output path, the second param `%s`, is the URL. The default is `curl -L -o %s %s`. - -## **--filename, -f** -Path to a configuration file. - -# EXAMPLES -Init a local registry from a specified configuration file. -``` -$ kk init registry -f config-example.yaml -``` -Init a local registry from a specified configuration file and use a KubeKey artifact. -``` -$ kk init registry -f config-example.yaml -a kubekey-artifact.tar.gz -``` - diff --git a/docs/commands/kk-init.md b/docs/commands/kk-init.md deleted file mode 100644 index 3b2818b1..00000000 --- a/docs/commands/kk-init.md +++ /dev/null @@ -1,11 +0,0 @@ -# NAME -**kk init**: Initializes the installation environment. - -# DESCRIPTION -Initializes the installation environment. - -# COMMANDS -| Command | Description | -| - | - | -| [kk init os](./kk-init-os.md) | Init operating system. | -| [kk init registry](./kk-init-registry.md) | Init a local image registry. | \ No newline at end of file diff --git a/docs/commands/kk-plugin-list.md b/docs/commands/kk-plugin-list.md deleted file mode 100644 index e9262175..00000000 --- a/docs/commands/kk-plugin-list.md +++ /dev/null @@ -1,17 +0,0 @@ -# NAME -**kk plugin list**: List all visible plugin executables on a user's PATH. - -# DESCRIPTION -Renew a cluster certs. - -# OPTIONS - -## **--name-only** -If true, display only the binary name of each plugin, rather than its full path. - -# EXAMPLES -``` -$ kk plugin list -``` - - diff --git a/docs/commands/kk-plugin.md b/docs/commands/kk-plugin.md deleted file mode 100644 index 7edd9c71..00000000 --- a/docs/commands/kk-plugin.md +++ /dev/null @@ -1,10 +0,0 @@ -# NAME -**kk plugin**: Provides utilities for interacting with plugins. - -# DESCRIPTION -Provides utilities for interacting with plugins. - -# COMMANDS -| Command | Description | -| - | - | -| [kk plugin list](./kk-plugin-list.md) | List all visible plugin executables on a user's PATH. | \ No newline at end of file diff --git a/docs/commands/kk-upgrade-phase.md b/docs/commands/kk-upgrade-phase.md deleted file mode 100644 index 929efb84..00000000 --- a/docs/commands/kk-upgrade-phase.md +++ /dev/null @@ -1,100 +0,0 @@ -# NAME -**kk alpha upgade phase**: Upgrade your cluster in phases to a newer version with this command, which is not enough and need to add phase cmds. - -# DESCRIPTION -Upgrade your cluster in phases to a newer version with this command, which is not enough and need to add phase cmds. - -# PHASES CMDS - -| Command | Description | -| - | - | -| kk alpha upgrade phase binary | Download the binary and synchronize kubernetes binaries. | -| kk alpha upgrade phase images | Pull the images before create your cluster. | -| kk alpha upgrade phase nodes | Upgrade cluster on master nodes and worker nodes to the version you input. | -| kk alpha upgrade phase kubesphere | Upgrade your kubesphere to a newer version with this command. | - -# OPTIONS - -## **--artifact, -a** -Path to a KubeKey artifact. - -## **--debug** -Print detailed information. The default is `false`. - -## **--download-cmd** -The user defined command to download the necessary binary files. The first param `%s` is output path, the second param `%s`, is the URL. The default is `curl -L -o %s %s`. - -## **--filename, -f** -Path to a configuration file. - -## **--ignore-err** -Ignore the error message, remove the host which reported error and force to continue. The default is `false`. - -## **--skip-pull-images** -Skip pre pull images. The default is `false`. - -## **--with-kubernetes** -Specify a supported version of kubernetes. It will override the version of kubernetes in the config file. - -## **--with-kubesphere** -Deploy a specific version of kubesphere. It will override the kubesphere `ClusterConfiguration` in the config file with the default value. - -## **--yes, -y** -Skip confirm check. The default is `false`. - -# EXAMPLES -## Upgrade an `all-in-one` Kubernetes cluster from a specified version -Upgrade the binarys of an `all-in-one` Kubernetes cluster from a specified version. -``` -$ kk alpha upgrade phase binary --with-kubernetes v1.23.8 -``` -Upgrade the images of an `all-in-one` Kubernetes cluster from a specified version. -``` -$ kk alpha upgrade phase images --with-kubernetes v1.23.8 -``` -Upgrade the cluster nodes of an `all-in-one` Kubernetes cluster from a specified version. -``` -$ kk alpha upgrade phase nodes --with-kubernetes v1.23.8 -``` -Upgrade the kubesphere of an `all-in-one` Kubernetes cluster from a specified version if you need. -``` -$ kk alpha upgrade phase kubesphere --with-kubesphere v3.3.0 -``` -## Upgrade an kubernetes cluster from a specified configuration file -Upgrade the binarys from a specified configuration file. -``` -$ kk alpha upgrade phase binary -f config-example.yaml -``` -Upgrade the images from a specified configuration file. -``` -$ kk alpha upgrade phase images -f config-example.yaml -``` -Upgrade the cluster nodes from a specified configuration file. -``` -$ kk alpha upgrade phase nodes -f config-example.yaml -``` -Upgrade the kubesphere from a specified configuration file if you need. -``` -$ kk alpha upgrade phase kubesphere -f config-example.yaml -``` -## Upgrade a cluster using a KubeKey artifact (in an offline enviroment). -import a KubeKey artifact named `my-artifact.tar.gz`. -``` -$ kk artifact import -a my-artifact.tar.gz -``` -Upgrade the binarys from a specified configuration file. -``` -$ kk alpha upgrade phase binary -f config-example.yaml -``` -Upgrade the images from a specified configuration file. -``` -$ kk alpha upgrade phase images -f config-example.yaml -``` -Upgrade the cluster nodes from a specified configuration file. -``` -$ kk alpha upgrade phase nodes -f config-example.yaml -``` -Upgrade the kubesphere from a specified configuration file if you need. -``` -$ kk alpha upgrade phase kubesphere -f config-example.yaml -``` \ No newline at end of file diff --git a/docs/commands/kk-upgrade.md b/docs/commands/kk-upgrade.md deleted file mode 100644 index ff836994..00000000 --- a/docs/commands/kk-upgrade.md +++ /dev/null @@ -1,48 +0,0 @@ -# NAME -**kk upgade**: Upgrade your cluster smoothly to a newer version with this command. - -# DESCRIPTION -Upgrade your cluster smoothly to a newer version with this command. - -# OPTIONS - -## **--artifact, -a** -Path to a KubeKey artifact. - -## **--debug** -Print detailed information. The default is `false`. - -## **--download-cmd** -The user defined command to download the necessary binary files. The first param `%s` is output path, the second param `%s`, is the URL. The default is `curl -L -o %s %s`. - -## **--filename, -f** -Path to a configuration file. - -## **--ignore-err** -Ignore the error message, remove the host which reported error and force to continue. The default is `false`. - -## **--skip-pull-images** -Skip pre pull images. The default is `false`. - -## **--with-kubernetes** -Specify a supported version of kubernetes. It will override the version of kubernetes in the config file. - -## **--with-kubesphere** -Deploy a specific version of kubesphere. It will override the kubesphere `ClusterConfiguration` in the config file with the default value. - -## **--yes, -y** -Skip confirm check. The default is `false`. - -# EXAMPLES -Upgrade an `all-in-one` Kubernetes cluster from a specified version. -``` -$ kk upgrade --with-kubernetes v1.22.0 -``` -Upgrade a cluster from a specified configuration file. -``` -$ kk upgrade -f config-example.yaml -``` -Upgrade a cluster using a KubeKey artifact (in an offline enviroment). -``` -$ kk upgrade -f config-example.yaml -a kubekey-artifact.tar.gz -``` \ No newline at end of file diff --git a/docs/commands/kk-version.md b/docs/commands/kk-version.md deleted file mode 100644 index 15b0af38..00000000 --- a/docs/commands/kk-version.md +++ /dev/null @@ -1,19 +0,0 @@ -# NAME -**kk version**: Print the client version information. - -# DESCRIPTION -Print the client version information. - -# OPTIONS - -## **--short** -Print the version number. - -## **--show-supported-k8s** -Print the version of supported k8s. - -# EXAMPLES -Print the current KubeKey client version. -``` -$ kk version -``` \ No newline at end of file diff --git a/docs/commands/kk.md b/docs/commands/kk.md deleted file mode 100644 index df5d1a3f..00000000 --- a/docs/commands/kk.md +++ /dev/null @@ -1,27 +0,0 @@ -# NAME -**kk**: Kubernetes/KubeSphere Deploy Tool - -# DESCRIPTION -KubeKey is an open-source lightweight tool for deploying Kubernetes clusters. It provides a flexible, rapid, and convenient way to install Kubernetes/K3s only, both Kubernetes/K3s and KubeSphere, and related cloud-native add-ons. It is also an efficient tool to scale and upgrade your cluster. - -In addition, KubeKey also supports customized Air-Gap package, which is convenient for users to quickly deploy clusters in offline environments. - -Use KubeKey in the following three scenarios. - -* Install Kubernetes/K3s only -* Install Kubernetes/K3s and KubeSphere together in one command -* Install Kubernetes/K3s first, then deploy KubeSphere on it using [ks-installer](https://github.com/kubesphere/ks-installer) - -# COMMANDS -| Command | Description | -| - | - | -| [kk add](./kk-add.md) | Add nodes to kubernetes cluster. | -| [kk artifact](./kk-artifact.md)| Manage a KubeKey offline installation package. | -| [kk certs](./kk-certs.md) | Manage cluster certs. | -| [kk completion](./kk-completion.md) | Generate shell completion scripts. | -| [kk create](./kk-create.md) | Create a cluster, a cluster configuration file or an offline installation package configuration file. | -| [kk delete](./kk-delete.md) | Delete node or cluster. | -| [kk init](./kk-init.md) | Initializes the installation environment. | -| [kk plugin](./kk-plugin.md) | Provides utilities for interacting with plugins. | -| [kk upgrade](./kk-upgrade.md) | Upgrade your cluster smoothly to a newer version with this command. | -| [kk version](./kk-version.md) | Print the client version information. | \ No newline at end of file diff --git a/docs/components-versions.md b/docs/components-versions.md deleted file mode 100644 index 16263531..00000000 --- a/docs/components-versions.md +++ /dev/null @@ -1,22 +0,0 @@ -## versions of default plugins and components used by kubekey -## based on apis/kubekey/v1alpha2 -| Components & Plugins | Version | -|-----------------------|------------------| -| etcd | v3.4.13 | -| docker | v20.10.8 | -| containerd | v1.6.4 | -| runc | v1.1.1 | -| crictl | v1.24.0 | -| kubernetes | v1.23.10 | -| calico | v3.23.2 | -| flannel | v0.12.0 | -| cni-plugins | v0.9.1 | -| cilium | v1.11.6 | -| kubeovn | v1.10.6 | -| multus | v3.8 | -| helm | v3.9.0 | -| docker-compose | v2.2.2 | -| docker-registy | v2 | -| Harbor | v2.5.3 | -| DPDK | v19.11 | - diff --git a/docs/config-example.md b/docs/config-example.md deleted file mode 100644 index 3a8796bb..00000000 --- a/docs/config-example.md +++ /dev/null @@ -1,288 +0,0 @@ -# Cluster Configuration Sample -```yaml -apiVersion: kubekey.kubesphere.io/v1alpha2 -kind: Cluster -metadata: - name: sample -spec: - hosts: - # Assume that the default port for SSH is 22. Otherwise, add the port number after the IP address. - # If you install Kubernetes on ARM, add "arch: arm64". For example, {...user: ubuntu, password: Qcloud@123, arch: arm64}. - - {name: node1, address: 172.16.0.2, internalAddress: 172.16.0.2, port: 8022, user: ubuntu, password: "Qcloud@123"} - # For default root user. - # Kubekey will parse `labels` field and automatically label the node. - - {name: node2, address: 172.16.0.3, internalAddress: 172.16.0.3, password: "Qcloud@123", labels: {disk: SSD, role: backend}} - # For password-less login with SSH keys. - - {name: node3, address: 172.16.0.4, internalAddress: 172.16.0.4, privateKeyPath: "~/.ssh/id_rsa"} - roleGroups: - etcd: - - node1 # All the nodes in your cluster that serve as the etcd nodes. - master: - - node1 - - node[2:10] # From node2 to node10. All the nodes in your cluster that serve as the master nodes. - worker: - - node1 - - node[10:100] # All the nodes in your cluster that serve as the worker nodes. - controlPlaneEndpoint: - # Internal loadbalancer for apiservers. Support: haproxy, kube-vip [Default: ""] - internalLoadbalancer: haproxy - # Determines whether to use external dns to resolve the control-plane domain. - # If 'externalDNS' is set to 'true', the 'address' needs to be set to "". - externalDNS: false - domain: lb.kubesphere.local - # The IP address of your load balancer. If you use internalLoadblancer in "kube-vip" mode, a VIP is required here. - address: "" - port: 6443 - system: - # The ntp servers of chrony. - ntpServers: - - time1.cloud.tencent.com - - ntp.aliyun.com - - node1 # Set the node name in `hosts` as ntp server if no public ntp servers access. - timezone: "Asia/Shanghai" - # Specify additional packages to be installed. The ISO file which is contained in the artifact is required. - rpms: - - nfs-utils - # Specify additional packages to be installed. The ISO file which is contained in the artifact is required. - debs: - - nfs-common - #preInstall: # Specify custom init shell scripts for each nodes, and execute according to the list order at the first stage. - # - name: format and mount disk - # bash: /bin/bash -x setup-disk.sh - # materials: # scripts can has some dependency materials. those will copy to the node - # - ./setup-disk.sh # the script which shell execute need - # - xxx # other tools materials need by this script - #postInstall: # Specify custom finish clean up shell scripts for each nodes after the Kubernetes install. - # - name: clean tmps files - # bash: | - # rm -fr /tmp/kubekey/* - #skipConfigureOS: true # Do not pre-configure the host OS (e.g. kernel modules, /etc/hosts, sysctl.conf, NTP servers, etc). You will have to set these things up via other methods before using KubeKey. - - kubernetes: - #kubelet start arguments - #kubeletArgs: - # Directory path for managing kubelet files (volume mounts, etc). - # - --root-dir=/var/lib/kubelet - version: v1.21.5 - # Optional extra Subject Alternative Names (SANs) to use for the API Server serving certificate. Can be both IP addresses and DNS names. - apiserverCertExtraSans: - - 192.168.8.8 - - lb.kubespheredev.local - # Container Runtime, support: containerd, cri-o, isula. [Default: docker] - containerManager: docker - clusterName: cluster.local - # Whether to install a script which can automatically renew the Kubernetes control plane certificates. [Default: false] - autoRenewCerts: true - # masqueradeAll tells kube-proxy to SNAT everything if using the pure iptables proxy mode. [Default: false]. - masqueradeAll: false - # maxPods is the number of Pods that can run on this Kubelet. [Default: 110] - maxPods: 110 - # podPidsLimit is the maximum number of PIDs in any pod. [Default: 10000] - podPidsLimit: 10000 - # The internal network node size allocation. This is the size allocated to each node on your network. [Default: 24] - nodeCidrMaskSize: 24 - # Specify which proxy mode to use. [Default: ipvs] - proxyMode: ipvs - # enable featureGates, [Default: {"ExpandCSIVolumes":true,"RotateKubeletServerCertificate": true,"CSIStorageCapacity":true, "TTLAfterFinished":true}] - featureGates: - CSIStorageCapacity: true - ExpandCSIVolumes: true - RotateKubeletServerCertificate: true - TTLAfterFinished: true - ## support kata and NFD - # kata: - # enabled: true - # nodeFeatureDiscovery - # enabled: true - # additional kube-proxy configurations - kubeProxyConfiguration: - ipvs: - # CIDR's to exclude when cleaning up IPVS rules. - # necessary to put node cidr here when internalLoadbalancer=kube-vip and proxyMode=ipvs - # refer to: https://github.com/kubesphere/kubekey/issues/1702 - excludeCIDRs: - - 172.16.0.2/24 - etcd: - # Specify the type of etcd used by the cluster. When the cluster type is k3s, setting this parameter to kubeadm is invalid. [kubekey | kubeadm | external] [Default: kubekey] - type: kubekey - ## The following parameters need to be added only when the type is set to external. - ## caFile, certFile and keyFile need not be set, if TLS authentication is not enabled for the existing etcd. - # external: - # endpoints: - # - https://192.168.6.6:2379 - # caFile: /pki/etcd/ca.crt - # certFile: /pki/etcd/etcd.crt - # keyFile: /pki/etcd/etcd.key - dataDir: "/var/lib/etcd" - # Time (in milliseconds) of a heartbeat interval. - heartbeatInterval: 250 - # Time (in milliseconds) for an election to timeout. - electionTimeout: 5000 - # Number of committed transactions to trigger a snapshot to disk. - snapshotCount: 10000 - # Auto compaction retention for mvcc key value store in hour. 0 means disable auto compaction. - autoCompactionRetention: 8 - # Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics. - metrics: basic - ## Etcd has a default of 2G for its space quota. If you put a value in etcd_memory_limit which is less than - ## etcd_quota_backend_bytes, you may encounter out of memory terminations of the etcd cluster. Please check - ## etcd documentation for more information. - # 8G is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it. - quotaBackendBytes: 2147483648 - # Maximum client request size in bytes the server will accept. - # etcd is designed to handle small key value pairs typical for metadata. - # Larger requests will work, but may increase the latency of other requests - maxRequestBytes: 1572864 - # Maximum number of snapshot files to retain (0 is unlimited) - maxSnapshots: 5 - # Maximum number of wal files to retain (0 is unlimited) - maxWals: 5 - # Configures log level. Only supports debug, info, warn, error, panic, or fatal. - logLevel: info - network: - plugin: calico - calico: - ipipMode: Always # IPIP Mode to use for the IPv4 POOL created at start up. If set to a value other than Never, vxlanMode should be set to "Never". [Always | CrossSubnet | Never] [Default: Always] - vxlanMode: Never # VXLAN Mode to use for the IPv4 POOL created at start up. If set to a value other than Never, ipipMode should be set to "Never". [Always | CrossSubnet | Never] [Default: Never] - vethMTU: 0 # The maximum transmission unit (MTU) setting determines the largest packet size that can be transmitted through your network. By default, MTU is auto-detected. [Default: 0] - kubePodsCIDR: 10.233.64.0/18 - kubeServiceCIDR: 10.233.0.0/18 - storage: - openebs: - basePath: /var/openebs/local # base path of the local PV provisioner - registry: - registryMirrors: [] - insecureRegistries: [] - privateRegistry: "" - namespaceOverride: "" - auths: # if docker add by `docker login`, if containerd append to `/etc/containerd/config.toml` - "dockerhub.kubekey.local": - username: "xxx" - password: "***" - skipTLSVerify: false # Allow contacting registries over HTTPS with failed TLS verification. - plainHTTP: false # Allow contacting registries over HTTP. - certsPath: "/etc/docker/certs.d/dockerhub.kubekey.local" # Use certificates at path (*.crt, *.cert, *.key) to connect to the registry. - addons: [] # You can install cloud-native addons (Chart or YAML) by using this field. - #dns: - # ## Optional hosts file content to coredns use as /etc/hosts file. - # dnsEtcHosts: | - # 192.168.0.100 api.example.com - # 192.168.0.200 ingress.example.com - # coredns: - # ## additionalConfigs adds any extra configuration to coredns - # additionalConfigs: | - # whoami - # log - # ## Array of optional external zones to coredns forward queries to. It's injected into coredns' config file before - # ## default kubernetes zone. Use it as an optimization for well-known zones and/or internal-only domains, i.e. VPN for internal networks (default is unset) - # externalZones: - # - zones: - # - example.com - # - example.io:1053 - # nameservers: - # - 1.1.1.1 - # - 2.2.2.2 - # cache: 5 - # - zones: - # - mycompany.local:4453 - # nameservers: - # - 192.168.0.53 - # cache: 10 - # - zones: - # - mydomain.tld - # nameservers: - # - 10.233.0.3 - # cache: 5 - # rewrite: - # - name substring website.tld website.namespace.svc.cluster.local - # ## Rewrite plugin block to perform internal message rewriting. - # rewriteBlock: | - # rewrite stop { - # name regex (.*)\.my\.domain {1}.svc.cluster.local - # answer name (.*)\.svc\.cluster\.local {1}.my.domain - # } - # ## DNS servers to be added *after* the cluster DNS. These serve as backup - # ## DNS servers in early cluster deployment when no cluster DNS is available yet. - # upstreamDNSServers: - # - 8.8.8.8 - # - 1.2.4.8 - # - 114.114.114.114 - # nodelocaldns: - # ## It's possible to extent the nodelocaldns' configuration by adding an array of external zones. - # externalZones: - # - zones: - # - example.com - # - example.io:1053 - # nameservers: - # - 1.1.1.1 - # - 2.2.2.2 - # cache: 5 - # - zones: - # - mycompany.local:4453 - # nameservers: - # - 192.168.0.53 - # cache: 10 - # - zones: - # - mydomain.tld - # nameservers: - # - 10.233.0.3 - # cache: 5 - # rewrite: - # - name substring website.tld website.namespace.svc.cluster.local - -``` - -# Network Configuration sample -## Hybridnet -To learn more about hybridnet, check out https://github.com/alibaba/hybridnet -```yaml - network: - plugin: hybridnet - hybridnet: - defaultNetworkType: Overlay - enableNetworkPolicy: false - init: false - preferVxlanInterfaces: eth0 - preferVlanInterfaces: eth0 - preferBGPInterfaces: eth0 - networks: - - name: "net1" - type: Underlay - nodeSelector: - network: "net1" - subnets: - - name: "subnet-10" - netID: 10 - cidr: "192.168.10.0/24" - gateway: "192.168.10.1" - - name: "subnet-11" - netID: 11 - cidr: "192.168.11.0/24" - gateway: "192.168.11.1" - - name: "net2" - type: Underlay - nodeSelector: - network: "net2" - subnets: - - name: "subnet-30" - netID: 30 - cidr: "192.168.30.0/24" - gateway: "192.168.30.1" - - name: "subnet-31" - netID: 31 - cidr: "192.168.31.0/24" - gateway: "192.168.31.1" - - name: "net3" - type: Underlay - netID: 0 - nodeSelector: - network: "net3" - subnets: - - name: "subnet-50" - cidr: "192.168.50.0/24" - gateway: "192.168.50.1" - start: "192.168.50.100" - end: "192.168.50.200" - reservedIPs: ["192.168.50.101","192.168.50.102"] - excludeIPs: ["192.168.50.111","192.168.50.112"] -``` diff --git a/docs/developer-guide.md b/docs/developer-guide.md deleted file mode 100644 index 247b4a57..00000000 --- a/docs/developer-guide.md +++ /dev/null @@ -1,30 +0,0 @@ -## Architecture - -KubeKey defines task pipelines for cluster operations such as installation, scaling, uninstallation, etc. And uses SSH and Kubernetes Api to perform corresponding tasks on hosts and cluster with host grouping and configuration management. - -![Image](img/KubeKey-Architecture.png?raw=true) - -* `Action`: The most basic unit. It represents one operation performed on one node; -* `Task`: The one that manages `Action`. It contains fields such as `Action`, `Hosts` (The hosts where the action will be executed), `Retry`, `Parallel`, etc. In short, it represents executing an `Action` on the specified hosts as required; -* `Module`: A unit that contains one or more related `Task`. `Module` is a module with specific and complete functions; -* `Pipeline`: It contains `Modules` combined in a certain order. It is the complete execution process of a KubeKey command. For example, `Create Cluster Pipeline`, `Add Nodes Pipeline` and so on. -## Addons -All plugins which are installed by yaml or chart can be kubernetes' addons. So the addons configuration support both yaml and chart. - -![Image](img/KubeKey-Addons.jpg?raw=true) - -The task of installing KubeSphere is added to the task pipeline of the installation cluster by default. So KubeSphere can be deployed in two ways: - -* Using the command `kk create cluster --with-kubesphere` -* Configure KubeSphere as a addon in the configuration file. - -> Notice: Installation of KubeSphere using [ks-installer](https://github.com/kubesphere/ks-installer). - - -## Build Binary from Source Code - -```shell script -git clone https://github.com/kubesphere/kubekey.git -cd kubekey -make kk -``` \ No newline at end of file diff --git a/docs/features.md b/docs/features.md deleted file mode 100644 index a7489551..00000000 --- a/docs/features.md +++ /dev/null @@ -1,48 +0,0 @@ -# Features - -### Cluster Lifecycle Management - -* Create clusters -* Delete clusters -* Add nodes -* Update control plane and worker nodes -* Renew cluster certificates - -### Supported Components - -- Core components - - Kubernetes/K3s - - etcd -- Container runtimes - - Docker - - containerd - - CRI-O (not integrated) - - iSula (not integrated) - - Kata -- Network plugins - - Calico - - Flannel - - Kube-OVN - - Cilium - - Multus CNI - - No plugin -- Storage - - OpenEBS Local PV - - Custom storage (allows users to customize storage service by using [addons](addons.md)) -- Container images registries - - [Docker registry](registry.md) - - [Harbor](registry.md) -- Applications - - Node Feature Discovery - -### Air-Gapped Installation - -- Customize and use the [air-gapped installation package](manifest_and_artifact.md) -- Create a private image registry -- Customize images available in the private image registry - -### Advanced Features - -- Custom system component configurations (kube-apiserver/kube-controller-manager/kube-scheduler/kubelet/kube-proxy) -- Command plugins -- [Universal task scheduling framework](developer-guide.md) \ No newline at end of file diff --git a/docs/glusterfs.yaml b/docs/glusterfs.yaml deleted file mode 100644 index 45930824..00000000 --- a/docs/glusterfs.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: heketi-secret - namespace: kube-system -type: kubernetes.io/glusterfs -data: - # e.g. jwt_admin_key: 123456 - key: SHOULD_BE_REPLACED - ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: glusterfs - annotations: - storageclass.kubesphere.io/supported-access-modes: '["ReadWriteOnce","ReadOnlyMany","ReadWriteMany"]' - # default storage - storageclass.beta.kubernetes.io/is-default-class: "false" -provisioner: kubernetes.io/glusterfs -parameters: - # e.g. glusterfs_provisioner_resturl: http://192.168.0.4:8080 - resturl: SHOULD_BE_REPLACED - # e.g. glusterfs_provisioner_clusterid: 6a6792ed25405eaa6302da99f2f5e24b - clusterid: SHOULD_BE_REPLACED - restauthenabled: "true" - restuser: admin - secretNamespace: "kube-system" - secretName: heketi-secret - gidMin: "40000" - gidMax: "50000" - volumetype: "replicate:2" -allowVolumeExpansion: true \ No newline at end of file diff --git a/docs/ha-mode.md b/docs/ha-mode.md deleted file mode 100644 index 9e245757..00000000 --- a/docs/ha-mode.md +++ /dev/null @@ -1,24 +0,0 @@ -# HA mode (internal loadbalancing) -K8s components require a loadbalancer to access the apiservers via a reverse proxy. Kubekey uses **kube-vip** and **haproxy** to provide internal ha mode. -## haproxy -The way kubekey uses is referred to as localhost loadbalancing. The kubelet of each master node connects the local kube-apiserver, and the kubelet of each worker node connects the kube-apiserver via a local reverse proxy. Based on this, kubekey will deploy a haproxy-based proxy that resides on each worker node as the local reverse proxy. - -![Image](img/haproxy.png?raw=true) - -## kube-vip -The load balancing is provided through IPVS (IP Virtual Server) and provides a Layer 4 (TCP-based) round-robin across all of the control plane nodes. By default, the load balancer will listen on the default port of 6443 as the Kubernetes API server. The IPVS virtual server lives in kernel space and doesn't create an "actual" service that listens on port 6443. This allows the kernel to parse packets before they're sent to an actual TCP port. Based on this, kubekey will deploy a static pod that resides on each control-plane node as the internal loadbalancing. - -![Image](img/kube-vip.png?raw=true) - -## Usage -Modify your configuration file and uncomment the item `internalLoadbalancer`: -```yaml -controlPlaneEndpoint: - internalLoadbalancer: haproxy #Internal loadbalancer for apiservers. Support: haproxy, kube-vip [Default: ""] - - domain: lb.kubesphere.local - address: "" # The IP address of your load balancer. If you use internalLoadblancer in "kube-vip" mode, a VIP is required here. - port: 6443 -``` - -Then whether you exec the command `create cluster`, `add nodes` or `upgrade`, kubekey will enable HA mode and deploy the interanl load balancer. \ No newline at end of file diff --git a/docs/harbor-ha.md b/docs/harbor-ha.md deleted file mode 100644 index 41397a42..00000000 --- a/docs/harbor-ha.md +++ /dev/null @@ -1,229 +0,0 @@ -## 一、Harbor 简介 - -Harbor 是由 VMware 公司使用 Go 语言开发，主要就是用于存放镜像使用，同时我们还可以通过 Web 界面来对存放的镜像进行管理。并且 Harbor 提供的功能有：基于角色的访问控制，镜像远程复制同步，以及审计日志等功能。官方文档 - -### 1.Harbor 功能介绍 - -1）基于角色的访问控制： 我们可以通过项目来对用户进行权限划分，项目中可以包含多个镜像。 - -2）审计管理： 我们可以在用户审计管理中，找到我们所有对镜像仓库做的操作。 - -3）镜像复制： 我们可以通过配置，使在两台 Harbor 服务器间实现镜像同步。 - -4）漏洞扫描： Harbor 会定期对仓库中的镜像进行扫描，并进行策略检查，以防止部署出易受到攻击的镜像。 - -### 2.Harbor 高可用方式 - -目前 Harbor 最常见的高可用方式有两种，分别是： - -1）安装两台 Harbor 仓库，他们共同使用一个存储（一般常见的便是 NFS 共享存储） - -![Image](img/harbor_ha01.png?raw=true) - -2）安装两台 Harbor 仓库，并互相配置同步关系。 - -![image](img/harbor_ha02.png?raw=true) - -因为第一种方式的话，需要额外配置 Redis 和 PostgreSQL 以及 NFS 服务，所以我们下面使用第二种方式进行 Harbor 高可用配置。 - -# Harbor镜像仓库高可用方案设计 - -采用2台harbor仓库互为主备的方案，如下图所示 - -![image](img/harbor_keepalived.png?raw=true) - -注意：主备方案 - -由于VIP的浮动，主备节点其实是互为主备；在部署harbor时，需要注意主备节点上的harbor.yml中hostname不要配置为浮动IP（reg.harbor.4a或192.168.10.200），应配置为各自IP或者hostname； -早先，将VIP的域名reg.harbor.4a配置到190和191上的harbor.yml中（hostname: reg.harbor.4a）导致一个问题：只有主节点可做为target被添加，用作镜像同步（也就是无法在主节点的仓库管理中创建备节点的target，即便添加了也无法连通）。 - -准备文件(在源码的script文件夹下,keepalived镜像可以从互联网阿里云镜像仓库下载)如下 - -``` -# tree . -. -├── harborCreateRegistriesAndReplications.sh -├── keepalived21.tar -├── kk -└── harbor_keepalived - ├── check_harbor.sh - ├── docker-compose-keepalived-backup.yaml - ├── docker-compose-keepalived-master.yaml - ├── keepalived-backup.conf - └── keepalived-master.conf - -1 directory, 8 files -``` - -kk: kubekey支持多节点harbor仓库代码(包含本pr)编译生成二进制文件 - -harborCreateRegistriesAndReplications.sh：配置harbor互为主备的脚本 - -keepalived21.tar：keepalived的docker镜像 - -harbor_keepalived：keepalived master和slave的docker-compose部署文件 - -## kubekey部署多节点harbor仓库 - -通过二次开发kubekey源码，实现了kubekey部署harbor仓库支持多节点，并且配置同一套harbor证书。证书中包含所有部署harbor节点的主机名和IP认证设置。 - -后续集成到一键部署脚本中，通过配置registry角色的多个节点来部署多harbor仓库。推荐2个harbor仓库，部署过多占用资源。 - -## harbor仓库互为主备设置 - -harbor仓库部署后，通过调用harbor仓库api建立备份仓库，建立备份规则。 - -例如master1节点上仓库和master2节点仓库配置如下 - -``` -#!/bin/bash - -Harbor_master1_Address=master1:7443 -master1_Address=192.168.122.61 -Harbor_master2_Address=master2:7443 -master2_Address=192.168.122.62 - -Harbor_User=admin #登录Harbor的用户 -Harbor_Passwd="Harbor12345" #登录Harbor的用户密码 -Harbor_UserPwd="$Harbor_User:$Harbor_Passwd" - -# create registry -curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master1_Address}/api/v2.0/registries" -d "{\"name\": \"master1_2_master2\", \"type\": \"harbor\", \"url\":\"https://${master2_Address}:7443\", \"credential\": {\"access_key\": \"${Harbor_User}\", \"access_secret\": \"${Harbor_Passwd}\"}, \"insecure\": true}" -# create registry -curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master2_Address}/api/v2.0/registries" -d "{\"name\": \"master2_2_master1\", \"type\": \"harbor\", \"url\":\"https://${master1_Address}:7443\", \"credential\": {\"access_key\": \"${Harbor_User}\", \"access_secret\": \"${Harbor_Passwd}\"}, \"insecure\": true}" - -#createReplication -curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master1_Address}/api/v2.0/replication/policies" -d "{\"name\": \"master1_2_master2\", \"enabled\": true, \"deletion\":true, \"override\":true, \"replicate_deletion\":true, \"dest_registry\":{ \"id\": 1, \"name\": \"master1_2_master2\"}, \"trigger\": {\"type\": \"event_based\"}, \"dest_namespace_replace_count\":1 }" - -#createReplication -curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master2_Address}/api/v2.0/replication/policies" -d "{\"name\": \"master2_2_master1\", \"enabled\": true, \"deletion\":true, \"override\":true, \"replicate_deletion\":true, \"dest_registry\":{ \"id\": 1, \"name\": \"master2_2_master1\"}, \"trigger\": {\"type\": \"event_based\"}, \"dest_namespace_replace_count\":1 }" -``` - -## keepalived管理harbor服务VIP - -使用docker-compose管理keepalived服务 - -keepalived master服务器配置如下 - -``` -# cat docker-compose-keepalived-master.yaml -version: '3.8' - -# Docker-Compose 单容器使用参考 YAML 配置文件 -# 更多配置参数请参考镜像 README.md 文档中说明 -services: - keepalived: - image: 'dockerhub.kubekey.local/kubesphere/keepalived:2.1' - privileged: true - network_mode: host - volumes: - - ./keepalived-master.conf:/srv/conf/keepalived/keepalived.conf - - ./check_harbor.sh:/srv/conf/keepalived/check_harbor.sh - container_name: keepalived - restart: on-failure - -# cat keepalived-master.conf -vrrp_script check_harbor { - script "/srv/conf/keepalived/check_harbor.sh" - interval 10 # 间隔时间，单位为秒，默认1秒 - fall 2 # 脚本几次失败转换为失败 - rise 2 # 脚本连续监测成功后，把服务器从失败标记为成功的次数 - timeout 5 - init_fail -} -global_defs { - script_user root - router_id harbor-ha - enable_script_security - lvs_sync_daemon ens3 VI_1 -} -vrrp_instance VI_1 { - state MASTER - interface ens3 - virtual_router_id 31 # 如果同一个局域网中有多套keepalive，那么要保证该id唯一 - priority 100 - advert_int 1 - authentication { - auth_type PASS - auth_pass k8s-test - } - virtual_ipaddress { - 192.168.122.59 - } - track_script { - check_harbor - } -} -# cat check_harbor.sh -#!/bin/bash -#count=$(docker-compose -f /opt/harbor/docker-compose.yml ps -a|grep healthy|wc -l) -# 不能频繁调用docker-compose 否则会有非常多的临时目录被创建：/tmp/_MEI* -count=$(docker ps |grep goharbor|grep healthy|wc -l) -status=$(ss -tlnp|grep -w 443|wc -l) -if [ $count -ne 11 -a ];then - exit 8 -elif [ $status -lt 2 ];then - exit 9 -else - exit 0 -fi -``` - -keepalived slave服务器跟master区别配置如下 - -1、state BACKUP 与 MASTER - -2、priority master配置为100，slave设置为50 - -``` -# cat docker-compose-keepalived-backup.yaml -version: '3.8' - -# Docker-Compose 单容器使用参考 YAML 配置文件 -# 更多配置参数请参考镜像 README.md 文档中说明 -services: - keepalived: - image: 'dockerhub.kubekey.local/kubesphere/keepalived:2.1' - privileged: true - network_mode: host - volumes: - - ./keepalived-backup.conf:/srv/conf/keepalived/keepalived.conf - - ./check_harbor.sh:/srv/conf/keepalived/check_harbor.sh - container_name: keepalived - restart: on-failure - -# cat keepalived-backup.conf -vrrp_script check_harbor { - script "/srv/conf/keepalived/check_harbor.sh" - interval 10 # 间隔时间，单位为秒，默认1秒 - fall 2 # 脚本几次失败转换为失败 - rise 2 # 脚本连续监测成功后，把服务器从失败标记为成功的次数 - timeout 5 - init_fail -} -global_defs { - script_user root - router_id harbor-ha - enable_script_security - lvs_sync_daemon ens3 VI_1 -} -vrrp_instance VI_1 { - state BACKUP - interface ens3 - virtual_router_id 31 # 如果同一个局域网中有多套keepalive，那么要保证该id唯一 - priority 50 - advert_int 1 - authentication { - auth_type PASS - auth_pass k8s-test - } - virtual_ipaddress { - 192.168.122.59 - } - track_script { - check_harbor - } -} -``` - -经常需要变动的参数是设置keepalived的interface和vip地址值，实际环境下可以参数化keepalived这2个值。 \ No newline at end of file diff --git a/docs/img/KubeKey-Addons.jpg b/docs/img/KubeKey-Addons.jpg deleted file mode 100644 index 876dac37..00000000 Binary files a/docs/img/KubeKey-Addons.jpg and /dev/null differ diff --git a/docs/img/KubeKey-Architecture.jpg b/docs/img/KubeKey-Architecture.jpg deleted file mode 100644 index aa99a92f..00000000 Binary files a/docs/img/KubeKey-Architecture.jpg and /dev/null differ diff --git a/docs/img/KubeKey-Architecture.png b/docs/img/KubeKey-Architecture.png deleted file mode 100644 index 2ec7834e..00000000 Binary files a/docs/img/KubeKey-Architecture.png and /dev/null differ diff --git a/docs/img/autoscaler-deployment.png b/docs/img/autoscaler-deployment.png deleted file mode 100644 index 1992cacc..00000000 Binary files a/docs/img/autoscaler-deployment.png and /dev/null differ diff --git a/docs/img/autoscaler-mechanism.png b/docs/img/autoscaler-mechanism.png deleted file mode 100644 index b177b2cc..00000000 Binary files a/docs/img/autoscaler-mechanism.png and /dev/null differ diff --git a/docs/img/haproxy.png b/docs/img/haproxy.png deleted file mode 100644 index cafe6ad3..00000000 Binary files a/docs/img/haproxy.png and /dev/null differ diff --git a/docs/img/harbor_ha01.png b/docs/img/harbor_ha01.png deleted file mode 100644 index 3778881f..00000000 Binary files a/docs/img/harbor_ha01.png and /dev/null differ diff --git a/docs/img/harbor_ha02.png b/docs/img/harbor_ha02.png deleted file mode 100644 index d64fb62c..00000000 Binary files a/docs/img/harbor_ha02.png and /dev/null differ diff --git a/docs/img/harbor_keepalived.png b/docs/img/harbor_keepalived.png deleted file mode 100644 index 5f001681..00000000 Binary files a/docs/img/harbor_keepalived.png and /dev/null differ diff --git a/docs/img/kube-vip.png b/docs/img/kube-vip.png deleted file mode 100644 index 9fa796b6..00000000 Binary files a/docs/img/kube-vip.png and /dev/null differ diff --git a/docs/img/kubekey-logo.svg b/docs/img/kubekey-logo.svg deleted file mode 100644 index c57298ef..00000000 --- a/docs/img/kubekey-logo.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - \ No newline at end of file diff --git a/docs/img/kubekey.svg b/docs/img/kubekey.svg deleted file mode 100644 index b4f4a214..00000000 --- a/docs/img/kubekey.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - \ No newline at end of file diff --git a/docs/k3s-versions.md b/docs/k3s-versions.md deleted file mode 100644 index b12d2628..00000000 --- a/docs/k3s-versions.md +++ /dev/null @@ -1,18 +0,0 @@ -## K3s Versions(amd64) - -| Version | Supported | -|----------|---------------------| -| v1.20.2 | :white_check_mark: | -| v1.20.4 | :white_check_mark: | -| v1.21.4 | :white_check_mark: | -| v1.21.6 | :white_check_mark: | -| v1.23.13 | :white_check_mark: | -| v1.24.7 | :white_check_mark: | - -## K3s Versions(arm64) -| Version | Supported | -|----------|--------------------| -| v1.21.4 | :white_check_mark: | -| v1.21.6 | :white_check_mark: | -| v1.23.13 | :white_check_mark: | -| v1.24.7 | :white_check_mark: | diff --git a/docs/k8e-versions.md b/docs/k8e-versions.md deleted file mode 100644 index 2fd65ed3..00000000 --- a/docs/k8e-versions.md +++ /dev/null @@ -1,15 +0,0 @@ -## What is K8E -Kubernetes Easy (k8e) is a lightweight, Extensible, Enterprise Kubernetes distribution that allows users to uniformly manage, secure, and get out-of-the-box kubernetes cluster for enterprise environments. more infors can be found in [K8E Site](https://getk8e.com/) - -## K8e Versions(amd64) - -| Version | Supported | -|----------|---------------------| -| v1.21.14 | :white_check_mark: | -| v1.25.5 | :white_check_mark: | - -## K8e Versions(arm64) -| Version | Supported | -|----------|--------------------| -| v1.21.14 | :white_check_mark: | -| v1.25.5 | :white_check_mark: | diff --git a/docs/kubectl-autocompletion.md b/docs/kubectl-autocompletion.md deleted file mode 100644 index 7c14062e..00000000 --- a/docs/kubectl-autocompletion.md +++ /dev/null @@ -1,19 +0,0 @@ -Enable kubectl autocompletion ------------- - -KubeKey doesn't enable kubectl autocompletion. Refer to the guide below and turn it on: - -**Prerequisite**: make sure bash-autocompletion is installed and works. - -```shell script -# Install bash-completion -apt-get install bash-completion - -# Source the completion script in your ~/.bashrc file -echo 'source <(kubectl completion bash)' >>~/.bashrc - -# Add the completion script to the /etc/bash_completion.d directory -kubectl completion bash >/etc/bash_completion.d/kubectl -``` - -More detail reference could be found [here](https://kubernetes.io/docs/tasks/tools/install-kubectl/#enabling-shell-autocompletion). \ No newline at end of file diff --git a/docs/kubekey-autocompletion.md b/docs/kubekey-autocompletion.md deleted file mode 100644 index 4ed127fe..00000000 --- a/docs/kubekey-autocompletion.md +++ /dev/null @@ -1,25 +0,0 @@ -Enable kubekey auto-completion ------------- - -**Prerequisite**: make sure `bash-completion` is installed and does workk. - -## Bash Completion - -You can install it via your distribution's package manager. - -For Ubuntu, `apt-get install bash-completion` - -For CentOS, `yum install -y bash-completion` - -## Setup Completion - -Write bash completion code to a file and source if from .bash_profile. - -``` -mkdir -p ~/.config/kk/ && kk completion --type bash > ~/.config/kk/completion.bash.inc -printf " -# kk shell completion -source '$HOME/.config/kk/completion.bash.inc' -" >> $HOME/.bash_profile -source $HOME/.bash_profile -``` diff --git a/docs/kubernetes-versions.md b/docs/kubernetes-versions.md deleted file mode 100644 index b1ed5a72..00000000 --- a/docs/kubernetes-versions.md +++ /dev/null @@ -1,126 +0,0 @@ -## Kubernetes Versions(amd64/arm64) -| Version | Supported | -|----------|--------------------| -| v1.19.0 | :white_check_mark: | -| v1.19.8 | :white_check_mark: | -| v1.19.9 | :white_check_mark: | -| v1.19.15 | :white_check_mark: | -| v1.20.4 | :white_check_mark: | -| v1.20.6 | :white_check_mark: | -| v1.20.10 | :white_check_mark: | -| v1.21.0 | :white_check_mark: | -| v1.21.1 | :white_check_mark: | -| v1.21.2 | :white_check_mark: | -| v1.21.3 | :white_check_mark: | -| v1.21.4 | :white_check_mark: | -| v1.21.5 | :white_check_mark: | -| v1.21.6 | :white_check_mark: | -| v1.21.7 | :white_check_mark: | -| v1.21.8 | :white_check_mark: | -| v1.21.9 | :white_check_mark: | -| v1.21.10 | :white_check_mark: | -| v1.21.11 | :white_check_mark: | -| v1.21.12 | :white_check_mark: | -| v1.21.13 | :white_check_mark: | -| v1.21.14 | :white_check_mark: | -| v1.22.0 | :white_check_mark: | -| v1.22.1 | :white_check_mark: | -| v1.22.2 | :white_check_mark: | -| v1.22.3 | :white_check_mark: | -| v1.22.4 | :white_check_mark: | -| v1.22.5 | :white_check_mark: | -| v1.22.6 | :white_check_mark: | -| v1.22.7 | :white_check_mark: | -| v1.22.8 | :white_check_mark: | -| v1.22.9 | :white_check_mark: | -| v1.22.10 | :white_check_mark: | -| v1.22.11 | :white_check_mark: | -| v1.22.12 | :white_check_mark: | -| v1.22.13 | :white_check_mark: | -| v1.22.14 | :white_check_mark: | -| v1.22.15 | :white_check_mark: | -| v1.22.16 | :white_check_mark: | -| v1.22.17 | :white_check_mark: | -| v1.23.0 | :white_check_mark: | -| v1.23.1 | :white_check_mark: | -| v1.23.2 | :white_check_mark: | -| v1.23.3 | :white_check_mark: | -| v1.23.4 | :white_check_mark: | -| v1.23.5 | :white_check_mark: | -| v1.23.6 | :white_check_mark: | -| v1.23.7 | :white_check_mark: | -| v1.23.8 | :white_check_mark: | -| v1.23.9 | :white_check_mark: | -| v1.23.10 | :white_check_mark: | -| v1.23.11 | :white_check_mark: | -| v1.23.12 | :white_check_mark: | -| v1.23.13 | :white_check_mark: | -| v1.23.14 | :white_check_mark: | -| v1.23.15 | :white_check_mark: | -| v1.23.16 | :white_check_mark: | -| v1.23.17 | :white_check_mark: | -| v1.24.0 | :white_check_mark: | -| v1.24.1 | :white_check_mark: | -| v1.24.2 | :white_check_mark: | -| v1.24.3 | :white_check_mark: | -| v1.24.4 | :white_check_mark: | -| v1.24.5 | :white_check_mark: | -| v1.24.6 | :white_check_mark: | -| v1.24.7 | :white_check_mark: | -| v1.24.8 | :white_check_mark: | -| v1.24.9 | :white_check_mark: | -| v1.24.10 | :white_check_mark: | -| v1.24.11 | :white_check_mark: | -| v1.24.12 | :white_check_mark: | -| v1.24.13 | :white_check_mark: | -| v1.24.14 | :white_check_mark: | -| v1.24.15 | :white_check_mark: | -| v1.24.16 | :white_check_mark: | -| v1.24.17 | :white_check_mark: | -| v1.25.0 | :white_check_mark: | -| v1.25.1 | :white_check_mark: | -| v1.25.2 | :white_check_mark: | -| v1.25.3 | :white_check_mark: | -| v1.25.4 | :white_check_mark: | -| v1.25.5 | :white_check_mark: | -| v1.25.6 | :white_check_mark: | -| v1.25.7 | :white_check_mark: | -| v1.25.8 | :white_check_mark: | -| v1.25.9 | :white_check_mark: | -| v1.25.10 | :white_check_mark: | -| v1.25.11 | :white_check_mark: | -| v1.25.12 | :white_check_mark: | -| v1.25.13 | :white_check_mark: | -| v1.25.14 | :white_check_mark: | -| v1.25.15 | :white_check_mark: | -| v1.25.16 | :white_check_mark: | -| v1.26.0 | :white_check_mark: | -| v1.26.1 | :white_check_mark: | -| v1.26.2 | :white_check_mark: | -| v1.26.3 | :white_check_mark: | -| v1.26.4 | :white_check_mark: | -| v1.26.5 | :white_check_mark: | -| v1.26.6 | :white_check_mark: | -| v1.26.7 | :white_check_mark: | -| v1.26.8 | :white_check_mark: | -| v1.26.9 | :white_check_mark: | -| v1.26.10 | :white_check_mark: | -| v1.26.11 | :white_check_mark: | -| v1.26.12 | :white_check_mark: | -| v1.27.0 | :white_check_mark: | -| v1.27.1 | :white_check_mark: | -| v1.27.2 | :white_check_mark: | -| v1.27.3 | :white_check_mark: | -| v1.27.4 | :white_check_mark: | -| v1.27.5 | :white_check_mark: | -| v1.27.6 | :white_check_mark: | -| v1.27.7 | :white_check_mark: | -| v1.27.8 | :white_check_mark: | -| v1.27.9 | :white_check_mark: | -| v1.28.0 | :white_check_mark: | -| v1.28.1 | :white_check_mark: | -| v1.28.2 | :white_check_mark: | -| v1.28.3 | :white_check_mark: | -| v1.28.4 | :white_check_mark: | -| v1.28.5 | :white_check_mark: | -| v1.29.0 | :white_check_mark: | diff --git a/docs/machinehealthcheck.md b/docs/machinehealthcheck.md deleted file mode 100644 index ed557b98..00000000 --- a/docs/machinehealthcheck.md +++ /dev/null @@ -1,27 +0,0 @@ -# HealthCheck for capkk - -refer https://cluster-api.sigs.k8s.io/tasks/automated-machine-management/healthchecking.html - -there is a sample for healthcheck - -```yaml -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineHealthCheck -metadata: - name: hc-capkk-1 -spec: - clusterName: capkk-1 - maxUnhealthy: 100% - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: capkk-1 - unhealthyConditions: - - type: Ready - status: Unknown - timeout: 300s - - type: Ready - status: "False" - timeout: 300s -``` - -Capkk currently does not have a remediationTemplate. \ No newline at end of file diff --git a/docs/manifest-example.md b/docs/manifest-example.md deleted file mode 100644 index 81668141..00000000 --- a/docs/manifest-example.md +++ /dev/null @@ -1,153 +0,0 @@ -# QuickStart -The following is an example of a manifest file for a kubernetes v1.21.5 cluster. It contains the repositories for `ubuntu 20.04` and `centos 7`, some necessary components, private registry, and necessary images. -```yaml -apiVersion: kubekey.kubesphere.io/v1alpha2 -kind: Manifest -metadata: - name: sample -spec: - arches: - - amd64 - operatingSystems: - - arch: amd64 - type: linux - id: ubuntu - version: "20.04" - osImage: Ubuntu 20.04.3 LTS - repository: - iso: - localPath: - url: https://github.com/kubesphere/kubekey/releases/download/v2.0.0/ubuntu-20.04-amd64-debs.iso - - arch: amd64 - type: linux - id: centos - version: "7" - osImage: CentOS Linux 7 (Core) - repository: - iso: - localPath: - url: https://github.com/kubesphere/kubekey/releases/download/v2.0.0/centos-7-amd64-rpms.iso - kubernetesDistributions: - - type: kubernetes - version: v1.21.5 - components: - helm: - version: v3.6.3 - cni: - version: v0.9.1 - etcd: - version: v3.4.13 - calicoctl: - version: v3.23.2 - containerRuntimes: - - type: docker - version: 20.10.8 - crictl: - version: v1.22.0 - docker-registry: - version: "2" - harbor: - version: v2.4.1 - docker-compose: - version: v2.2.2 - images: - - docker.io/calico/cni:v3.23.2 - - docker.io/calico/kube-controllers:v3.23.2 - - docker.io/calico/node:v3.23.2 - - docker.io/calico/pod2daemon-flexvol:v3.23.2 - - docker.io/coredns/coredns:1.8.0 - - docker.io/kubesphere/k8s-dns-node-cache:1.15.12 - - docker.io/kubesphere/kube-apiserver:v1.21.5 - - docker.io/kubesphere/kube-controller-manager:v1.21.5 - - docker.io/kubesphere/kube-proxy:v1.21.5 - - docker.io/kubesphere/kube-scheduler:v1.21.5 - - docker.io/kubesphere/pause:3.4.1 -``` - -# The Manifest Definition -The following is a full fields definition of the manifest file. -```yaml -apiVersion: kubekey.kubesphere.io/v1alpha2 -kind: Manifest -metadata: - name: sample -spec: - arches: # Define the architecture that will be included in the artifact. - - amd64 - - arm64 - operatingSystems: # Define the operating system that will be included in the artifact. - - arch: amd64 - type: linux - id: ubuntu - version: "20.04" - osImage: Ubuntu 20.04.3 LTS - repository: # Define the operating system repository iso file that will be included in the artifact. - iso: - localPath: ./ubuntu.iso # Define getting the iso file from the local path. - url: # Define getting the iso file from the URL. - - arch: amd64 - type: linux - id: centos - version: "7" - osImage: CentOS Linux 7 (Core) - repository: - iso: - localPath: - url: https://github.com/kubesphere/kubekey/releases/download/v2.0.0/centos-7-amd64-rpms.iso - kubernetesDistributions: # Define the kubernetes distribution that will be included in the artifact. - - type: kubernetes - version: v1.21.5 - - type: kubernetes - version: v1.22.1 - ## The following components' versions are automatically generated based on the default configuration of KubeKey. - components: - helm: - version: v3.6.3 - cni: - version: v0.9.1 - etcd: - version: v3.4.13 - ## For now, if your cluster container runtime is containerd, KubeKey will add a docker 20.10.8 container runtime in the below list. - ## The reason is KubeKey creates a cluster with containerd by installing a docker first and making kubelet connect the socket file of containerd which docker contained. - containerRuntimes: - - type: docker - version: 20.10.8 - crictl: - version: v1.22.0 - ## The following components define the private registry files that will be included in the artifact. - docker-registry: - version: "2" - harbor: - version: v2.4.1 - docker-compose: - version: v2.2.2 - ## Define the images that will be included in the artifact. - ## When you generate this file using KubeKey, all the images contained on the cluster hosts will be automatically added. - ## Of course, you can also modify this list of images manually. - images: - - docker.io/calico/cni:v3.20.0 - - docker.io/calico/kube-controllers:v3.20.0 - - docker.io/calico/node:v3.20.0 - - docker.io/calico/pod2daemon-flexvol:v3.20.0 - - docker.io/coredns/coredns:1.8.0 - - docker.io/kubesphere/k8s-dns-node-cache:1.15.12 - - docker.io/kubesphere/kube-apiserver:v1.21.5 - - docker.io/kubesphere/kube-controller-manager:v1.21.5 - - docker.io/kubesphere/kube-proxy:v1.21.5 - - docker.io/kubesphere/kube-scheduler:v1.21.5 - - docker.io/kubesphere/pause:3.4.1 - - dockerhub.kubekey.local/kubesphere/kube-apiserver:v1.22.1 - - dockerhub.kubekey.local/kubesphere/kube-controller-manager:v1.22.1 - - dockerhub.kubekey.local/kubesphere/kube-proxy:v1.22.1 - - dockerhub.kubekey.local/kubesphere/kube-scheduler:v1.22.1 - - dockerhub.kubekey.local/kubesphere/pause:3.5 - ## Define the authentication information if you need to pull images from a registry that requires authorization. - registry: - auths: - "dockerhub.kubekey.local": - username: "xxx" - password: "***" - skipTLSVerify: false # Allow contacting registries over HTTPS with failed TLS verification. - plainHTTP: false # Allow contacting registries over HTTP. - certsPath: "/etc/docker/certs.d/dockerhub.kubekey.local" # Use certificates at path (*.crt, *.cert, *.key) to connect to the registry. -``` diff --git a/docs/manifest_and_artifact.md b/docs/manifest_and_artifact.md deleted file mode 100644 index 0354fa0f..00000000 --- a/docs/manifest_and_artifact.md +++ /dev/null @@ -1,84 +0,0 @@ -# KubeKey Manifest And Artifact -KubeKey v2.0.0 (hereinafter kk) adds the concepts of `manifest` and `artifact` to provide a solution for users to deploy Kubernetes clusters offline enviroment. In the past, users had to prepare deployment tools, images' `tar` files, and other related binaries, and each user has a different version of Kubernetes to deploy and different images to deploy. Now with kk, you only need to use the `manifest` file to define what you need for the cluster environment to be deployed offline, and then use that `manifest` to export the `artifact` file to complete the preparation. Then offline installation requires only kk and `artifact` for quick and easy deployment of image registry (docker-registry or harbor) and Kubernetes clusters in your environment. - -## What is the KubeKey Manifest? -The `Manifest` is an offline installation package configuration file. There are currently two ways to generate this file： -* Manually creating and writing the file from a template. -* Generate the file from an existing cluster using the kk command. - -The first way requires more information about the different fields in this configuration file, see [manifest-example.yaml](./manifest-example.md). - -The following is only for the second way of generating files using the kk command. - -### Usage -> Note: -> This way requires preparing a cluster environment that has been installed a Kubernetes cluster and providing the `./kube/config` file of the cluster for kk. - -Commands: -``` -./kk create manifest -``` -By default, kk will use the `$HOME/.kube/config` file, or you can specify a `config` file: -``` -./kk create manifest --kubeconfig config -``` -After execution, the `manifest-sample.yaml` file will be generated in the current directory. The contents of the `manifest-sample.yaml` file can then be modified to export the desired `artifact` file later. - -### Principle -kk connects to the corresponding Kubernetes cluster via the `kubeconfig` file and then checks out the following information in the cluster environment: -* Node architecture -* Node operating system -* The images on the node -* Kubernetes version -* CRI information - -After that, the description of the current cluster will be written to the `manifest` file. Besides, other undetectable files (e.g. ETCD cluster information, image regsitry, etc.) will be written to the `manifest` file according to the default values recommended by kk. - -## What is the KubeKey Artifact? -The `artifact` is an offline installation package, exported from the specified `manifest` file. An `artifact` can be specified in the kk `init registry`, `create cluster`, `add node` and `upgrade cluster` commands. kk will automatically unpack the `artifact` and will use the unpacked file directly when executing the command. - -### Usage -#### Export Artifact -> Note: -> 1. The export command will download the corresponding binaries from the Internet, so please make sure the network connection is success. -> 2. kk will parse the image's name in the image list, if the mirror in the image's name needs authentication information, you can configure it in the `.registry.auths` field in the `manifest` file. -> 3. If the `artifact` file to be exported contains OS dependency files (e.g. conntarck, chrony, etc.), you can configure the corresponding ISO dependency download URL address in the `.repostiory.iso.url` in the `operationSystems` field. - -* Export -``` -./kk artifact export -m manifest-sample.yaml -``` -After execution, the `kubekey-artifact.tar.gz` file will be generated in the current directory. - -#### Use Artifact -> Note: -> 1. In an offline environment, you need to use kk to generate the `config-sample.yaml` file and configure the corresponding information before using the `artifact`. -> 2. In an offline environment, the `artifact` image will be pushed to the private image registry by default when you use the `create cluster` and `upgrade cluster` commands. If the private image registry needs authentication information, you can configure it in the `.spec.registry.auths` field in the `config-sample.yaml` file. - -* Initialize the image registry, the related configuration can be found in [container image registry](./registry.md). - ``` -./kk init registry -f config-sample.yaml -a kubekey-artifact.tar.gz -``` -* Push the image to the private image registry. -``` -./kk artifact image push -f config-sample.yaml -a kubekey-artifact.tar.gz -``` -* Create the cluster. -> Note: In an offline environment, you need to configure private image registry information for cluster image management, please refer to [config-sample.yaml](./config-example.md) and [container image registry](./registry.md). - -``` -./kk create cluster -f config-sample.yaml -a kubekey-artifact.tar.gz -``` - -* Create the cluster and install the OS dependencies (requires the `artifact` to contain the OS dependency files for the nodes in the target cluster). -``` -./kk create cluster -f config-sample.yaml -a kubekey-artifact.tar.gz --with-packages -``` -* Add nodes. -``` -./kk add nodes -f config-sample.yaml -a kubekey-artifact.tar.gz -``` -* Upgrade the cluster. -``` -./kk upgrade -f config-sample.yaml -a kubekey-artifact.tar.gz -``` \ No newline at end of file diff --git a/docs/network-access.md b/docs/network-access.md deleted file mode 100644 index b3a21c0d..00000000 --- a/docs/network-access.md +++ /dev/null @@ -1,19 +0,0 @@ -Network Access ------------- -If your network configuration uses an firewall，you must ensure infrastructure components can communicate with each other through specific ports that act as communication endpoints for certain processes or services. - -|services|protocol|action|start port|end port|comment -|---|---|---|---|---|---| -ssh|TCP|allow|22| -etcd|TCP|allow|2379|2380| -apiserver|TCP|allow|6443| -calico|TCP|allow|9099|9100| -bgp|TCP|allow|179|| -nodeport|TCP|allow|30000|32767| -master|TCP|allow|10250|10258| -dns|TCP|allow|53| -dns|UDP|allow|53| -local-registry|TCP|allow|5000||offline environment| -local-apt|TCP|allow|5080||offline environment| -rpcbind|TCP|allow|111|| use NFS -ipip|IPENCAP / IPIP|allow| | |calico needs to allow the ipip protocol \ No newline at end of file diff --git a/docs/network-configurations.md b/docs/network-configurations.md deleted file mode 100644 index f47c3d22..00000000 --- a/docs/network-configurations.md +++ /dev/null @@ -1,25 +0,0 @@ -### Network Configurations - -#### IPVS - -If your cluster's kubeProxy mode is `ipvs` which is default value in `kk`, kubernetes will add some ipvs records on each node. You can use `ipvsadm` command to get more information. - -#### Iptables - -If your cluster's kubeProxy mode is `iptables`, kubernetes will add some iptables records on each node. You can use `iptables` command to get more information. - -#### Virtual Device - -Most of CNI Plugins will create some virtual devices on each node, such as `cni0`. You can use `ip link` command to inspect them in details. - -As for `flannel`, virtual devices named with `flannel` prefix will be created. -As for `calico`, virtual devices named in `cali[a-f0-9]*` regexp format will be created. -As for `cilium`, virtual devices named with `cilium_` prefix will be created. - -If your cluster's kubeProxy mode is `ipvs`, additional virtual device `kube-ipvs0` will be created. - -If your cluster enables `nodelocaldns` feature for DNS caching purpose, additional virtual device `nodelocaldns` will be created. - -#### Network Namespace - -CNI plugins may create some network namespaces named with `cni-` prefix depends on which CNI plugin you choose to use. You can use `ip netns show 2>/dev/null | grep cni-` command to get CNI network namespace list. diff --git a/docs/registry.md b/docs/registry.md deleted file mode 100644 index 892bf6ff..00000000 --- a/docs/registry.md +++ /dev/null @@ -1,63 +0,0 @@ -# Container Image Registry - -KubeKey now allows users to create a container image registry ([Docker registry](https://docs.docker.com/registry/) or [Harbor](https://goharbor.io/)). - -### How to Use - -1. Initialize a local images registry: - - ``` - ./kk init registry [(-f | --filename) path] - ``` - -2. Add information about the server used as the image registry to **spec.hosts** and **spec.roleGroups** in the KubeKey configuration file. - - Example: - - ``` - apiVersion: kubekey.kubesphere.io/v1alpha2 - kind: Cluster - metadata: - name: sample - spec: - hosts: - - {name: node1, address: 192.168.6.6, internalAddress: 192.168.6.6, password: Qcloud@123} - roleGroups: - etcd: - - node1 - control-plane: - - node1 - worker: - - node1 - ## Specify the node role as registry. Only one node can be set as registry. - registry: - - node1 - controlPlaneEndpoint: - ##Internal loadbalancer for apiservers - #internalLoadbalancer: haproxy - domain: lb.kubesphere.local - address: "" - port: 6443 - kubernetes: - version: v1.21.5 - clusterName: cluster.local - network: - plugin: calico - kubePodsCIDR: 10.233.64.0/18 - kubeServiceCIDR: 10.233.0.0/18 - # multus support. https://github.com/k8snetworkplumbingwg/multus-cni - enableMultusCNI: false - registry: - ## `docker registry` is used to create local registry by default. - ## `harbor` can be also set for type. - # type: "harbor" - privateRegistry: dockerhub.kubekey.local - auths: - "dockerhub.kubekey.local": - username: admin - password: Harbor12345 - registryMirrors: [] - insecureRegistries: [] - addons: [] - ``` - diff --git a/docs/roadmap.md b/docs/roadmap.md deleted file mode 100644 index f12af17d..00000000 --- a/docs/roadmap.md +++ /dev/null @@ -1,40 +0,0 @@ -# KubeKey's RoadMap - -KubeKey Roadmap demonstrates a list of open source product development plans, as well as KubeSphere community's anticipation. Obviously, it details the future's direction of KubeKey, but may change over time. We hope that can help you to get familiar with the project plans and vision through the Roadmap. Of course, if you have any better ideas, welcome to filing Issues. -## v1.1 -### Feature: -- [x] Support for deploying k8s in ARM architecture. -- [x] Support more container runtimes: cri-o containerd isula -- [x] Support more cni plugins: cilium kube-ovn -- [x] Support for deploying clusters without cni plugin. -- [x] Support custom components parameters. -- [x] Support certificate expiration check and certificate update. -- [x] Support backup of etcd data. -- [x] Support for adding labels to nodes when deploying cluster. -- [ ] Support for adding taints to nodes when deploying cluster. -- [x] Support command auto-completion. -- [x] Support for deploying k3s (experimental). - -## v1.2.0 -### Feature: -- [x] Support for deploying Highly Available clusters by using internal load balancer. -- [x] Support Kubernetes certificate automatic renew. - -## v2.0.0 -### Feature: -- [x] More flexible task scheduling framework. -- [x] Support easier and more flexible air-gapped installation. -- [x] Support kubekey to independently generate certificate. -- [x] Support custom private registry authorization. -- [x] Enable featureGates in Kubernetes of cluster-config. -- [x] Support Kata and Node Feature Discovery. -- [x] Support customizing dnsDomain for the cluster. -- [x] Add pod PID limit and PID available. -- [x] Support setting NTP server and timezone. - -## v2.1.0 -### Feature: -- [] Support the use of kubeadm to manage etcd and use of existing etcd. -- [] Optimize the Container Manager installation process. -- [] Reduce the size of KubeKey artifact. -- [] Support more version of Kubernetes. \ No newline at end of file diff --git a/docs/storage-client.md b/docs/storage-client.md deleted file mode 100644 index 7b44beca..00000000 --- a/docs/storage-client.md +++ /dev/null @@ -1,52 +0,0 @@ -# Storage Client -## NFS -```shell script -# Debian / Ubuntu -apt install nfs-common - -# Centos / Redhat -yum install nfs-utils -``` -> Recommended nfs server configuration: *(rw,insecure,sync,no_subtree_check,no_root_squash) -## Ceph -```shell script -# Debian / Ubuntu -apt install ceph-common - -# Centos / Redhat -yum install ceph-common -``` -## GlusterFS - - * The following kernel modules must be loaded: - - 1. dm_snapshot - 2. dm_mirror - 3. dm_thin_pool - - For kernel modules, `lsmod | grep ` will show you if a given module is present, and `modprobe ` will load - a given module. - - * Each node requires that the `mount.glusterfs` command is available. - - * GlusterFS client version installed on nodes should be as close as possible to the version of the server. - * Take `glusterfs 7.x` as an example. -```shell script -# Debian -wget -O - https://download.gluster.org/pub/gluster/glusterfs/7/rsa.pub | apt-key add - -DEBID=$(grep 'VERSION_ID=' /etc/os-release | cut -d '=' -f 2 | tr -d '"') -DEBVER=$(grep 'VERSION=' /etc/os-release | grep -Eo '[a-z]+') -DEBARCH=$(dpkg --print-architecture) -echo deb https://download.gluster.org/pub/gluster/glusterfs/LATEST/Debian/${DEBID}/${DEBARCH}/apt ${DEBVER} main > /etc/apt/sources.list.d/gluster.list -apt update -apt install glusterfs-client - -# Ubuntu -apt install software-properties-common -add-apt-repository ppa:gluster/glusterfs-7 -apt update -apt install glusterfs-client - -# Centos / Redhat -yum install glusterfs-fuse -``` diff --git a/docs/turn-off-SELinux.md b/docs/turn-off-SELinux.md deleted file mode 100644 index a14c377d..00000000 --- a/docs/turn-off-SELinux.md +++ /dev/null @@ -1,21 +0,0 @@ -# How to turn off SELinux -## turn off SELinux -```shell script -# Edit the configuration -sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config -#restart the system -reboot -# check SELinux -getenforce -``` -> Edit the configuration file /etc/selinux/config, change SELINUX= to SELINUX=disabled, then restart the system, SELinux will be disabled - -## Temporarily shut down SELinux -```shell script -# Temporarily closing SELinux is to switch between enforcing and permissive modes -setenforce 0 #Switch to tolerance mode -setenforce 1 #Switch to mandatory mode -# check SELinux -getenforce -``` -> Temporary shutdown enforcing, invalid after restarting the system diff --git a/docs/turn-off-SELinux_zh-CN.md b/docs/turn-off-SELinux_zh-CN.md deleted file mode 100644 index 9a389e99..00000000 --- a/docs/turn-off-SELinux_zh-CN.md +++ /dev/null @@ -1,21 +0,0 @@ -# 如何关闭SELinux -## 永久关闭SELinux -```shell script -# 永久关闭SELinux -sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config -#关闭后需要重启系统 -reboot -# 查看SELinux的状态 -getenforce -``` -> 编辑配置文件/etc/selinux/config，把 SELINUX= 更改为 SELINUX=disabled ，然后重启系统，SELinux 就被禁用了 - -## 临时关闭SELinux -```shell script -# 临时关闭SELinux就是enforcing 和 permissive 两种模式之间进行切换 -setenforce 0 #切换成宽容模式 -setenforce 1 #切换成强制模式 -# check SELinux -getenforce -``` -> 临时切换工作模式，重启系统生失效 diff --git a/docs/zh/CAPKK开发文档.md b/docs/zh/CAPKK开发文档.md deleted file mode 100644 index eed3d24c..00000000 --- a/docs/zh/CAPKK开发文档.md +++ /dev/null @@ -1,182 +0,0 @@ -# CAPKK 开发者文档 - -Cluster-API-provider-kubekey（简称：CAPKK）是遵循 cluster-api 的开发者文档 [Provider contracts](https://cluster-api.sigs.k8s.io/developer/providers/contracts.html#provider-contract) 实现的基于 SSH 的基础设施供应商（ infrastructure provider ）。 - -在整个 CAPKK 项目中主要包含了三种资源及其对应的 controller：KKCluster，KKMachine，KKInstance。 - -## 项目结构 - -```bash -├── api -│ └── v1beta1 ## CAPKK 定义的 CR -├── bootstrap ## k3s 版本的 bootstrap provider -├── config ## CAPKK 项目的 Kubernetes yaml 文件 -├── controllers ## CAPKK 包含的 controller -│ ├── kkcluster -│ ├── kkinstance -│ └── kkmachine -├── controlplane ## k3s 版本的 control-plane provider -├── docs ## 文档 -├── exp ## 实验性功能 -├── hack -├── pkg -│ ├── clients ## CAPKK 用到的客户端 -│ │ └── ssh -│ ├── rootfs ## CAPKK 的数据存储目录 -│ ├── scope ## CAPKK 中按照功能划分定义的不同范围的接口，主要用于拆分 KKCluster、KKMachine、KKInstance 的属性字段。 -│ ├── service ## CAPKK 中各种操作的具体实现 -│ │ ├── binary ## CAPKK 对二进制文件的操作 -│ │ ├── bootstrap ## CAPKK 对机器环境初始化的操作 -│ │ ├── containermanager ## CAPKK 对容器运行时的操作 -│ │ ├── operation ## CAPKK 对 Linux 机器的基础操作 -│ │ ├── provisioning ## CAPKK 对 cloud-init 或 ignition 文件进行解析并映射为 SSH 命令 -│ │ ├── repository ## CAPKK 对 rpm 包的操作 -│ │ └── util -│ └── util -│ ├── filesystem -│ └── hash -├── scripts -├── templates ## 用于自动化脚本生成 GitHub release 中的 sample yaml -├── test ## CAPKK 的 e2e 测试 -│ └── e2e -├── util ## KubeKey 的通用工具类 -└── version ## 包含 KubeKey 的 version 和二进制组件的 SHA256 - -``` - -## KKCluster - -KKCluster 用于定义集群的全局的基本信息。 - -```go -type KKClusterSpec struct { - // 定义 Kubernetes 的发行版，目前支持的参数有 kubernetes，k3s - Distribution string `json:"distribution,omitempty"` - - // 定义节点的 SSH 认证信息 - Nodes Nodes `json:"nodes"` - - // 定义控制平面的端点 - // 可选 - ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` - - // 定义控制平面 LoadBalancer 的地址 - ControlPlaneLoadBalancer *KKLoadBalancerSpec `json:"controlPlaneLoadBalancer,omitempty"` - - // 定义集群组件二进制文件的下载地址，通常用于覆写默认地址，离线部署场景 - // 可选 - Component *Component `json:"component,omitempty"` - - // 定义私有镜像仓库信息 - // 可选 - Registry Registry `json:"registry,omitempty"` -} -``` - -在上述内容中，需注意如下几点： -* nodes 字段填写集群所包含的所有节点的 SSH 信息 -* CAPKK 默认使用 kube-vip 实现集群控制平面高可用，因此 controlPlaneLoadBalancer 可填写为网段内的任意为使用的 IP 。 -* CAPKK 默认使用二进制组件的官方下载地址，如需使用 KubeSphere 团队维护的国内资源地址可按如下方式配置： - ```yaml - spec: - component: - zone: cn - ``` -* component 字段还可用于指定自定义的文件服务器地址，并可配置 overrides 数组来修改 CAPKK 在文件服务器中对不同二进制的寻址 path 路径和 SHA256 checksum。 - -以下为一个常见 kkcluster 的 sample yaml 文件： -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKCluster -metadata: - name: quick-start -spec: - component: - zone: "" - nodes: - auth: - user: ubuntu - password: P@88w0rd - instances: - - name: test1 - address: 192.168.0.3 - - name: test2 - address: 192.168.0.4 - - name: test3 - address: 192.168.0.5 - controlPlaneLoadBalancer: - host: 192.168.0.100 -``` - -在 apply 该 yaml 文件后，KKCluster 的 webhook 会添加一些默认值，如 `roles: [controle-plane, worker]`。 - -## KKMachine - -KKMachine 主要用于表示 Kubernetes node 节点和基础设施主机之间的映射。 - -```go -type KKMachineSpec struct { - // 基础设施供应商的独有 ID - ProviderID *string `json:"providerID,omitempty"` - - // 对应的 KKInstance 资源的名称 - InstanceID *string `json:"instanceID,omitempty"` - - // 该资源的角色 - // 可选 - Roles []Role `json:"roles"` - - // 该资源对应节点的容器运行时 - // 可选 - ContainerManager ContainerManager `json:"containerManager"` - - // 该资源对应节点的 Linux 软件包 - // 可选 - Repository *Repository `json:"repository,omitempty"` -} -``` - -通常我们不直接使用 KKMachine 资源，而是声明 KKMachineTemplate 资源供 cluster-api 的资源进行引用。 - -### KKMachineTemplate - -所谓的 xxxTemplate 资源就是该类资源的模版类型，其包含了对应资源的 spec 中的所有内容。在 cluster-api 的使用过程中，我们会声明 KubeadmControlPlane（KCP）、MachineDeployment（MD）这样的带有 replica 字段资源，即说明了该类资源可进行动态伸缩。而这些资源中又包含了 infrastructureRef 字段，用于引用具体的基础设施模版资源（xxxMachineTemplate）。当发生扩容时，KCP、MD 这些资源就会根据模版资源创建出对应的，真正的，单个该资源实例。 - -以下为一个常见的控制平面 KKMachineTemplate 的 sample yaml 例子： -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKMachineTemplate -metadata: - name: quick-start - namespace: default -spec: - template: - spec: - roles: - - control-plane - containerManager: - type: containerd - repository: - iso: "none" - update: false -``` - -以下为注意事项： -* roles 角色指定 control-plane，即引用该 KKMachineTemplate 的资源只会选择 KKCluster 中对应 role 的机器进行引导。 -* containerManager 指定该模版创建的 KKMachine 配置容器运行时为 containerd。 -* repository 指定该模版创建的 KKMachine 操作 rpm 软件源的策略，该事例中的策略表示 CAPKK 会使用对应 Linux 机器上的默认软件源安装默认需要软件依赖（ conntrack，socat 等）。 - -## KKInstance - -在 cluster-api 的开发者文档中对于 infra 的定义仅包含 xxxCluster 和 xxxMachine 资源，而 KKInstance 是 CAPKK 独有的资源。CAPKK 额外定义 KKInstance 的目的是解耦 operator 和 controller 的逻辑，即 KKMachine 专注于维护于 cluster-api CR 交互，而 KKInstance 专注于维护 Linux 机器（通过 SSH 对机器进行命令式的操作）。 - -KKInstance 的属性字段主要就是 KKCluster 中 nodes 字段和 KKMachine 字段的集合，这里不做赘述。 - -KKInstance 的 controller 对于机器的操作归纳起来主要有以下几部分： -* Bootstrop：CAPKK 中定义的一些对机器的环境初始化或清理操作。如创建相应目录、添加 Linux 用户、运行初始化 shell 脚本、重置网络、删除文件、Kubeadm重置等操作。 -* Repository：CAPKK 中定义的处理 rpm 软件源的操作。如挂载 ISO 软件包，更新源，安装依赖软件包等操作。 -* Binary：CAPKK 中定义的处理集群组件二进制文件的操作。如下载二进制文件等。 -* ContainerManager：CAPKK 中定义的处理机器容器运行时的操作。如检查容器运行时，安装容器运行时等操作。 -* Provisioning：CAPKK 解析 cluster-api 提供的对应机器的 cloud-init 文件，并转换为 SSH 命令的操作。该 cloud-init 文件中就会包含诸如 “kubeadm init“、”kubeadm join“ 等操作。 - -对于这些操作的接口定义，见 [接口](https://github.com/kubesphere/kubekey/blob/master/pkg/service/interface.go)。 diff --git a/docs/zh/manifest_and_artifact.md b/docs/zh/manifest_and_artifact.md deleted file mode 100644 index ffb7ed39..00000000 --- a/docs/zh/manifest_and_artifact.md +++ /dev/null @@ -1,81 +0,0 @@ -# KubeKey 清单和制品 -KubeKey v2.0.0 版本（以下简称 kk ）新增了清单（manifest）和制品（artifact）的概念，为用户离线部署 Kubernetes 集群提供了一种解决方案。在过去，用户需要准备部署工具，镜像 tar 包和其他相关的二进制文件，每位用户需要部署的 Kubernetes 版本和需要部署的镜像都是不同的。现在使用 kk 的话，用户只需使用清单`manifest`文件来定义将要离线部署的集群环境需要的内容，再通过该`manifest`来导出制品`artifact`文件即可完成准备工作。离线部署时只需要 kk 和 `artifact`就可快速、简单的在环境中部署镜像仓库和 Kubernetes 集群。 - -## 什么是 KubeKey 清单 （manifest）？ -`manifest`就是一个描述当前 Kubernetes 集群信息和定义`artifact`制品中需要包含哪些内容的文本文件。目前有两种方式来生成该文件： -* 根据模版手动创建并编写该文件。 -* 使用 kk 命令根据已存在的集群生成该文件。 - -第一种方式需要了解该配置文件中不同字段的更多信息，请参考 [manifest-example.yaml](../manifest-example.md)。 - -以下内容仅针对于第二种由 kk 实现的生成文件的方式。 - -### 使用方式 -> 注意： -> 该方式需提前准备好一个已经搭建好的集群环境，并为 kk 提供集群的`kubeconfig`文件。 - -命令如下： -``` -./kk create manifest -``` -默认情况下 kk 会使用`$HOME/.kube/config`文件，也可指定一个`kubeconfig`文件。 -``` -./kk create manifest --kubeconfig config -``` -执行完毕后将在当前目录下生成`manifest-sample.yaml`文件。然后可根据实际情况修改`manifest-sample.yaml`文件的内容，用以之后导出期望的`artifact`文件。 - -### 原理 -kk 通过 `kubeconfig` 文件连接对应的 Kubernetes 集群，然后检查出集群环境中以下信息： -* 节点架构 -* 节点操作系统 -* 节点上的镜像 -* Kubernetes 版本 -* CRI 信息 - -之后，这些当前集群的描述信息将最终写入`manifest`文件中。同时其他无法检测到的相关文件信息（如：ETCD 集群信息、镜像仓库等）也将按照 kk 推荐的默认值写入到`manifest`文件中。 - -## 什么是 KubeKey 制品（artifact）？ -制品就是一个根据指定的`manifest`文件内容导出的包含镜像 tar 包和相关二进制文件的 tgz 包。在 kk 初始化镜像仓库、创建集群、添加节点和升级集群的命令中均可指定一个`artifact`，kk 将自动解包该`artifact`并将在执行命令时直接使用解包出来的文件。 - -### 使用方式 -#### 导出 artifact -> 注意： -> 1. 导出命令会从互联网中下载相应的二进制文件，请确保网络连接正常。 -> 2. kk 会解析镜像列表中的镜像名，若镜像名中的镜像仓库需要鉴权信息，可在`manifest`文件中的`.registry.auths`字段中进行配置。 -> 3. 若需要导出的`artifact`文件中包含操作系统依赖文件（如：conntarck、chrony等），可在`operationSystem`元素中的`.repository.iso.url`中配置相应的 ISO 依赖文件下载地址。 -* 导出。 -``` -./kk artifact export -m manifest-sample.yaml -``` -执行完毕后将在当前目录下生成`kubekey-artifact.tar.gz`文件。 - -#### 使用 artifact -> 注意： -> 1. 在离线环境中，使用`artifact`前需要先使用 kk 生成`config-sample.yaml`文件并配置相应的信息。 -> 2. 在离线环境中，使用创建集群和升级集群命令时默认情况下会将`artifact`中的镜像推送至私有镜像仓库，若私有镜像仓库需要鉴权信息，可在`config-sample.yaml`文件中的`.spec.registry.auths`字段中进行配置。 - -* 初始化镜像仓库，相关配置可参考 [容器镜像仓库](../registry.md)。 -``` -./kk init registry -f config-sample.yaml -a kubekey-artifact.tar.gz -``` -* 推送镜像到私有镜像仓库 -``` -./kk artifact image push -f config-sample.yaml -a kubekey-artifact.tar.gz -``` -* 创建集群。 -> 注意：在离线环境中，需配置私有镜像仓库信息用于集群镜像的管理，相关配置可参考 [config-sample.yaml](../config-example.md) 和 [容器镜像仓库](../registry.md)。 -``` -./kk create cluster -f config-sample.yaml -a kubekey-artifact.tar.gz -``` -* 创建集群，并且安装操作系统依赖（需要`artifact`中包含目标集群中节点的操作系统依赖文件）。 -``` -./kk create cluster -f config-sample.yaml -a kubekey-artifact.tar.gz --with-packages -``` -* 添加节点。 -``` -./kk add nodes -f config-sample.yaml -a kubekey-artifact.tar.gz -``` -* 升级集群。 -``` -./kk upgrade -f config-sample.yaml -a kubekey-artifact.tar.gz -``` diff --git a/exp/README.md b/exp/README.md deleted file mode 100644 index 3808b322..00000000 --- a/exp/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Experimental - -⚠️ This package holds experimental code and API types. ⚠️ - -## Compatibility notice - -This package does not adhere to any compatibility guarantees. Some portions may eventually be promoted out of this package and considered stable/GA, while others may be removed entirely. \ No newline at end of file diff --git a/go.mod b/go.mod deleted file mode 100644 index 0b0055c8..00000000 --- a/go.mod +++ /dev/null @@ -1,252 +0,0 @@ -module github.com/kubesphere/kubekey/v3 - -go 1.19 - -replace ( - github.com/docker/distribution => github.com/docker/distribution v2.8.1+incompatible - github.com/docker/docker => github.com/moby/moby v20.10.14+incompatible -) - -require ( - github.com/blang/semver v3.5.1+incompatible - github.com/containerd/containerd v1.6.10 - github.com/containers/image/v5 v5.21.1 - github.com/deckarep/golang-set v1.8.0 - github.com/estesp/manifest-tool/v2 v2.0.3 - github.com/evanphx/json-patch v5.6.0+incompatible - github.com/go-logr/logr v1.2.3 - github.com/google/go-cmp v0.5.9 - github.com/hashicorp/go-getter v1.6.2 - github.com/imdario/mergo v0.3.13 - github.com/jinzhu/copier v0.3.5 - github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible - github.com/lithammer/dedent v1.1.0 - github.com/mitchellh/mapstructure v1.5.0 - github.com/modood/table v0.0.0-20220527013332-8d47e76dad33 - github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.20.1 - github.com/opencontainers/image-spec v1.1.0-rc1 - github.com/pkg/errors v0.9.1 - github.com/pkg/sftp v1.13.5 - github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 - github.com/sirupsen/logrus v1.9.0 - github.com/spf13/cobra v1.5.0 - github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.12.0 - golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 - gopkg.in/yaml.v3 v3.0.1 - helm.sh/helm/v3 v3.9.4 - k8s.io/api v0.25.4 - k8s.io/apiextensions-apiserver v0.25.4 - k8s.io/apimachinery v0.25.4 - k8s.io/apiserver v0.25.4 - k8s.io/cli-runtime v0.25.4 - k8s.io/client-go v0.25.4 - k8s.io/cluster-bootstrap v0.25.4 - k8s.io/component-base v0.25.4 - k8s.io/klog/v2 v2.70.1 - k8s.io/kubectl v0.25.4 - k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed - sigs.k8s.io/cluster-api v1.2.6 - sigs.k8s.io/cluster-api/test v1.2.6 - sigs.k8s.io/controller-runtime v0.12.3 - sigs.k8s.io/kind v0.14.0 - sigs.k8s.io/yaml v1.3.0 -) - -require ( - cloud.google.com/go v0.104.0 // indirect - cloud.google.com/go/compute v1.7.0 // indirect - cloud.google.com/go/iam v0.5.0 // indirect - cloud.google.com/go/storage v1.23.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.28 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect - github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect - github.com/Azure/go-autorest/logger v0.2.1 // indirect - github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/BurntSushi/toml v1.2.0 // indirect - github.com/MakeNowJust/heredoc v1.0.0 // indirect - github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver/v3 v3.1.1 // indirect - github.com/Masterminds/sprig/v3 v3.2.2 // indirect - github.com/Masterminds/squirrel v1.5.3 // indirect - github.com/Microsoft/go-winio v0.5.2 // indirect - github.com/Microsoft/hcsshim v0.9.5 // indirect - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/VividCortex/ewma v1.2.0 // indirect - github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect - github.com/alessio/shellescape v1.4.1 // indirect - github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed // indirect - github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect - github.com/aws/aws-sdk-go v1.44.102 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect - github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/chai2010/gettext-go v1.0.2 // indirect - github.com/containerd/cgroups v1.0.4 // indirect - github.com/containerd/stargz-snapshotter/estargz v0.12.0 // indirect - github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a // indirect - github.com/containers/ocicrypt v1.1.5 // indirect - github.com/containers/storage v1.43.0 // indirect - github.com/coredns/caddy v1.1.0 // indirect - github.com/coredns/corefile-migration v1.0.17 // indirect - github.com/cyphar/filepath-securejoin v0.2.3 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/deislabs/oras v0.9.0 // indirect - github.com/docker/cli v20.10.17+incompatible // indirect - github.com/docker/distribution v2.8.1+incompatible // indirect - github.com/docker/docker v20.10.18+incompatible // indirect - github.com/docker/docker-credential-helpers v0.7.0 // indirect - github.com/docker/go-connections v0.4.1-0.20190612165340-fd1b1942c4d5 // indirect - github.com/docker/go-metrics v0.0.1 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect - github.com/emicklei/go-restful/v3 v3.8.0 // indirect - github.com/evanphx/json-patch/v5 v5.6.0 // indirect - github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect - github.com/fatih/camelcase v1.0.0 // indirect - github.com/fatih/color v1.13.0 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect - github.com/fvbommel/sortorder v1.0.1 // indirect - github.com/ghodss/yaml v1.0.0 // indirect - github.com/go-errors/errors v1.0.1 // indirect - github.com/go-gorp/gorp/v3 v3.0.2 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.19.5 // indirect - github.com/go-openapi/swag v0.19.14 // indirect - github.com/gobuffalo/flect v0.2.5 // indirect - github.com/gobwas/glob v0.2.3 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.4.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/google/btree v1.0.1 // indirect - github.com/google/cel-go v0.12.5 // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-github/v45 v45.2.0 // indirect - github.com/google/go-intervals v0.0.2 // indirect - github.com/google/go-querystring v1.1.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect - github.com/googleapis/gax-go/v2 v2.5.1 // indirect - github.com/googleapis/go-type-adapters v1.0.0 // indirect - github.com/gorilla/mux v1.8.0 // indirect - github.com/gosuri/uitable v0.0.4 // indirect - github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-safetemp v1.0.0 // indirect - github.com/hashicorp/go-version v1.6.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/huandu/xstrings v1.3.2 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/jmoiron/sqlx v1.3.5 // indirect - github.com/jonboulle/clockwork v0.2.2 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.15.11 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect - github.com/kr/fs v0.1.0 // indirect - github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect - github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect - github.com/lestrrat-go/strftime v1.0.5 // indirect - github.com/lib/pq v1.10.6 // indirect - github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect - github.com/magiconair/properties v1.8.6 // indirect - github.com/mailru/easyjson v0.7.6 // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect - github.com/mattn/go-shellwords v1.0.12 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/miekg/pkcs11 v1.1.1 // indirect - github.com/mistifyio/go-zfs/v3 v3.0.0 // indirect - github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/go-testing-interface v1.14.1 // indirect - github.com/mitchellh/go-wordwrap v1.0.0 // indirect - github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/moby/locker v1.0.1 // indirect - github.com/moby/spdystream v0.2.0 // indirect - github.com/moby/sys/mountinfo v0.6.2 // indirect - github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect - github.com/morikuni/aec v1.0.0 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/nxadm/tail v1.4.8 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/runc v1.1.4 // indirect - github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect - github.com/opencontainers/selinux v1.10.2 // indirect - github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f // indirect - github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.0.1 // indirect - github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/proglottis/gpgme v0.1.3 // indirect - github.com/prometheus/client_golang v1.12.2 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.7.3 // indirect - github.com/rivo/uniseg v0.2.0 // indirect - github.com/rubenv/sql-migrate v1.1.1 // indirect - github.com/russross/blackfriday v1.6.0 // indirect - github.com/shopspring/decimal v1.2.0 // indirect - github.com/smartystreets/goconvey v1.7.2 // indirect - github.com/spf13/afero v1.8.2 // indirect - github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 // indirect - github.com/stoewer/go-strcase v1.2.0 // indirect - github.com/subosito/gotenv v1.3.0 // indirect - github.com/sylabs/sif/v2 v2.8.0 // indirect - github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect - github.com/tchap/go-patricia v2.3.0+incompatible // indirect - github.com/ulikunitz/xz v0.5.10 // indirect - github.com/valyala/fastjson v1.6.3 // indirect - github.com/vbatts/tar-split v0.11.2 // indirect - github.com/vbauerster/mpb/v7 v7.5.3 // indirect - github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect - github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xeipuuv/gojsonschema v1.2.0 // indirect - github.com/xlab/treeprint v1.1.0 // indirect - go.etcd.io/bbolt v1.3.6 // indirect - go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect - go.opencensus.io v0.23.0 // indirect - go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - go.uber.org/atomic v1.9.0 // indirect - golang.org/x/net v0.1.0 // indirect - golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 // indirect - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/sys v0.1.0 // indirect - golang.org/x/term v0.1.0 // indirect - golang.org/x/text v0.4.0 // indirect - golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect - golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect - gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect - google.golang.org/api v0.97.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce // indirect - google.golang.org/grpc v1.49.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.66.4 // indirect - gopkg.in/square/go-jose.v2 v2.6.0 // indirect - gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect - oras.land/oras-go v1.2.0 // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect - sigs.k8s.io/kustomize/api v0.12.1 // indirect - sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect -) diff --git a/go.sum b/go.sum deleted file mode 100644 index a64458c9..00000000 --- a/go.sum +++ /dev/null @@ -1,2089 +0,0 @@ -bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= -bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= -cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= -cloud.google.com/go v0.104.0 h1:gSmWO7DY1vOm0MVU6DNXM11BWHHsTUmsC5cv1fuW5X8= -cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= -cloud.google.com/go/compute v1.7.0 h1:v/k9Eueb8aAJ0vZuxKMrgm6kPhCLZU9HxFU+AFDs9Uk= -cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= -cloud.google.com/go/iam v0.5.0 h1:fz9X5zyTWBmamZsqvqZqD7khbifcZF/q+Z1J8pfhIUg= -cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= -cloud.google.com/go/storage v1.23.0 h1:wWRIaDURQA8xxHguFCshYepGlrWIrbBnAmc7wfg07qY= -cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774 h1:SCbEWT58NSt7d2mcFdvxC9uyrdcTfvBbPLThhkDmXzg= -github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= -github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= -github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= -github.com/Azure/go-autorest/autorest/adal v0.9.20 h1:gJ3E98kMpFB1MFqQCvA1yFab8vthOeD4VlFRQULxahg= -github.com/Azure/go-autorest/autorest/adal v0.9.20/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= -github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= -github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= -github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= -github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= -github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= -github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= -github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= -github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= -github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= -github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= -github.com/Masterminds/squirrel v1.5.3 h1:YPpoceAcxuzIljlr5iWpNKaql7hLeG1KLSrhvdHpkZc= -github.com/Masterminds/squirrel v1.5.3/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= -github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= -github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= -github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= -github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= -github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= -github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= -github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= -github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= -github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= -github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= -github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= -github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= -github.com/Microsoft/hcsshim v0.9.4/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= -github.com/Microsoft/hcsshim v0.9.5 h1:AbV+VPfTrIVffukazHcpxmz/sRiE6YaMDzHWR9BXZHo= -github.com/Microsoft/hcsshim v0.9.5/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= -github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= -github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/ProtonMail/go-crypto v0.0.0-20220407094043-a94812496cf5/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= -github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= -github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= -github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= -github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= -github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed h1:ue9pVfIcP+QMEjfgo/Ez4ZjNZfonGgR6NgjMaJMu1Cg= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= -github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= -github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= -github.com/aws/aws-sdk-go v1.44.102 h1:6tUCTGL2UDbFZae1TLGk8vTgeXuzkb8KbAe2FiAeKHc= -github.com/aws/aws-sdk-go v1.44.102/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= -github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= -github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/bshuster-repo/logrus-logstash-hook v1.0.2 h1:JYRWo+QGnQdedgshosug9hxpPYTB9oJ1ZZD3fY31alU= -github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/bugsnag/bugsnag-go v2.1.2+incompatible h1:E7dor84qzwUO8KdCM68CZwq9QOSR7HXlLx3Wj5vui2s= -github.com/bugsnag/panicwrap v1.3.4 h1:A6sXFtDGsgU/4BLf5JT0o5uYg3EeKgGx3Sfs+/uk3pU= -github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= -github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= -github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= -github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= -github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= -github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= -github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= -github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= -github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= -github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= -github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= -github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= -github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= -github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= -github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= -github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= -github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= -github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= -github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= -github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= -github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= -github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= -github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= -github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= -github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= -github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= -github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= -github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= -github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= -github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= -github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= -github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= -github.com/containerd/containerd v1.6.1/go.mod h1:1nJz5xCZPusx6jJU8Frfct988y0NpumIq9ODB0kLtoE= -github.com/containerd/containerd v1.6.10 h1:8aiav7I2ZyQLbTlNMcBXyAU1FtFvp6VuyuW13qSd6Hk= -github.com/containerd/containerd v1.6.10/go.mod h1:CVqfxdJ95PDgORwA219AwwLrREZgrTFybXu2HfMKRG0= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= -github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= -github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= -github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= -github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= -github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= -github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= -github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= -github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= -github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= -github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= -github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= -github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= -github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= -github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= -github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= -github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= -github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= -github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4= -github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= -github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= -github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= -github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= -github.com/containerd/stargz-snapshotter/estargz v0.11.4/go.mod h1:7vRJIcImfY8bpifnMjt+HTJoQxASq7T28MYbP15/Nf0= -github.com/containerd/stargz-snapshotter/estargz v0.12.0 h1:idtwRTLjk2erqiYhPWy2L844By8NRFYEwYHcXhoIWPM= -github.com/containerd/stargz-snapshotter/estargz v0.12.0/go.mod h1:AIQ59TewBFJ4GOPEQXujcrJ/EKxh5xXZegW1rkR1P/M= -github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= -github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= -github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= -github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= -github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= -github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= -github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= -github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= -github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y= -github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= -github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= -github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE= -github.com/containers/image/v5 v5.21.1 h1:Cr3zw2f0FZs4SCkdGlc8SN/mpcmg2AKG4OUuDbeGS/Q= -github.com/containers/image/v5 v5.21.1/go.mod h1:zl35egpcDQa79IEXIuoUe1bW+D1pdxRxYjNlyb3YiXw= -github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a h1:spAGlqziZjCJL25C6F1zsQY05tfCKE9F5YwtEWWe6hU= -github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= -github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= -github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= -github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= -github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= -github.com/containers/ocicrypt v1.1.4-0.20220428134531-566b808bdf6f/go.mod h1:xpdkbVAuaH3WzbEabUd5yDsl9SwJA5pABH85425Es2g= -github.com/containers/ocicrypt v1.1.5 h1:UO+gBnBXvMvC7HTXLh0bPgLslfW8HlY+oxYcoSHBcZQ= -github.com/containers/ocicrypt v1.1.5/go.mod h1:WgjxPWdTJMqYMjf3M6cuIFFA1/MpyyhIM99YInA+Rvc= -github.com/containers/storage v1.40.0/go.mod h1:zUyPC3CFIGR1OhY1CKkffxgw9+LuH76PGvVcFj38dgs= -github.com/containers/storage v1.43.0 h1:P+zulGXA3mqe2GnYmZU0xu87Wy1M0PVHM2ucrgmvTdU= -github.com/containers/storage v1.43.0/go.mod h1:uZ147thiIFGdVTjMmIw19knttQnUCl3y9zjreHrg11s= -github.com/coredns/caddy v1.1.0 h1:ezvsPrT/tA/7pYDBZxu0cT0VmWk75AfIaf6GSYCNMf0= -github.com/coredns/caddy v1.1.0/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4= -github.com/coredns/corefile-migration v1.0.17 h1:tNwh8+4WOANV6NjSljwgW7qViJfhvPUt1kosj4rR8yg= -github.com/coredns/corefile-migration v1.0.17/go.mod h1:XnhgULOEouimnzgn0t4WPuFDN2/PJQcTxdWKC5eXNGE= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= -github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= -github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= -github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI= -github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= -github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= -github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= -github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= -github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= -github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= -github.com/deislabs/oras v0.9.0 h1:R6PRN3bTruUjHcGKgdteurzbpsCxwf3XJCLsxLFyBuU= -github.com/deislabs/oras v0.9.0/go.mod h1:QXnMi3+eEm/rkgGT6L+Lt0TT2WLA7pOzuk7tZIsUhFM= -github.com/denisenkom/go-mssqldb v0.9.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/distribution/distribution/v3 v3.0.0-20220526142353-ffbd94cbe269 h1:hbCT8ZPPMqefiAWD2ZKjn7ypokIGViTvBBg/ExLSdCk= -github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/cli v20.10.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/cli v20.10.17+incompatible h1:eO2KS7ZFeov5UJeaDmIs1NFEDRf32PaqRpvoEkKBy5M= -github.com/docker/cli v20.10.17+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= -github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c= -github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= -github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-connections v0.4.1-0.20190612165340-fd1b1942c4d5 h1:2o8D0hdBky229bNnc7a8bAZkeVMpH4qsp2Rmt4g/+Zk= -github.com/docker/go-connections v0.4.1-0.20190612165340-fd1b1942c4d5/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= -github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= -github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= -github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= -github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= -github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 h1:7QPwrLT79GlD5sizHf27aoY2RTvw62mO6x7mxkScNk0= -github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46/go.mod h1:esf2rsHFNlZlxsqsZDojNBcnNs5REqIvRrWRHqX0vEU= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= -github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/estesp/manifest-tool/v2 v2.0.3 h1:F9HMOqcXvtW+8drQB+BjNRU/+bLXOwCfj3mbjqQC2Ns= -github.com/estesp/manifest-tool/v2 v2.0.3/go.mod h1:Suh+tbKQvKHcs4Vltzy8gwZk1y9eSRI635gT4gFw5Ss= -github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= -github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= -github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= -github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= -github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= -github.com/fvbommel/sortorder v1.0.1 h1:dSnXLt4mJYH25uDDGa3biZNQsozaUWDSWeKJ0qqFfzE= -github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= -github.com/garyburd/redigo v1.6.3 h1:HCeeRluvAgMusMomi1+6Y5dmFOdYV/JzoRrrbFlkGIc= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gorp/gorp/v3 v3.0.2 h1:ULqJXIekoqMx29FI5ekXXFoH1dT2Vc8UhnRzBg+Emz4= -github.com/go-gorp/gorp/v3 v3.0.2/go.mod h1:BJ3q1ejpV8cVALtcXvXaXyTOlMmJhWDxTmncaR6rwBY= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= -github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gobuffalo/flect v0.2.5 h1:H6vvsv2an0lalEaCDRThvtBfmg44W/QHXBCYUXf/6S4= -github.com/gobuffalo/flect v0.2.5/go.mod h1:1ZyCLIbg0YD7sDkzvFdPoOydPtD8y9JQnrOROolUcM8= -github.com/gobuffalo/logger v1.0.6 h1:nnZNpxYo0zx+Aj9RfMPBm+x9zAU2OayFh/xrAWi34HU= -github.com/gobuffalo/logger v1.0.6/go.mod h1:J31TBEHR1QLV2683OXTAItYIg8pv2JMHnF/quuAbMjs= -github.com/gobuffalo/packd v1.0.1 h1:U2wXfRr4E9DH8IdsDLlRFwTZTK7hLfq9qT/QHXGVe/0= -github.com/gobuffalo/packd v1.0.1/go.mod h1:PP2POP3p3RXGz7Jh6eYEf93S7vA2za6xM7QT85L4+VY= -github.com/gobuffalo/packr/v2 v2.8.3 h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XEWlY= -github.com/gobuffalo/packr/v2 v2.8.3/go.mod h1:0SahksCVcx4IMnigTjiFuyldmTrdTctXsOdiU5KwbKc= -github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godror/godror v0.24.2/go.mod h1:wZv/9vPiUib6tkoDl+AZ/QLf5YZgMravZ7jxH2eQWAE= -github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= -github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= -github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= -github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/cel-go v0.12.5 h1:DmzaiSgoaqGCjtpPQWl26/gND+yRpim56H1jCVev6d8= -github.com/google/cel-go v0.12.5/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= -github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI= -github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28= -github.com/google/go-intervals v0.0.2 h1:FGrVEiUnTRKR8yE04qzXYaJMtnIYqobR5QbblK3ixcM= -github.com/google/go-intervals v0.0.2/go.mod h1:MkaR3LNRfeKLPmqgJYs4E66z5InYjmCjbbr4TQlcT6Y= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/enterprise-certificate-proxy v0.1.0 h1:zO8WHNx/MYiAKJ3d5spxZXZE6KHmIQGQcAzwUzV7qQw= -github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= -github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= -github.com/googleapis/gax-go/v2 v2.5.1 h1:kBRZU0PSuI7PspsSb/ChWoVResUcwNVIdpB049pKTiw= -github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/googleapis/go-type-adapters v1.0.0 h1:9XdMn+d/G57qq1s8dNc5IesGCXHf6V2HZ2JwRxfA2tA= -github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= -github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-getter v1.6.2 h1:7jX7xcB+uVCliddZgeKyNxv0xoT7qL5KDtH7rU4IqIk= -github.com/hashicorp/go-getter v1.6.2/go.mod h1:IZCrswsZPeWv9IkVnLElzRU/gz/QPi6pZHn4tv6vbwA= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= -github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= -github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= -github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= -github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= -github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= -github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= -github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= -github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= -github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= -github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.2/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= -github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kortschak/utter v1.0.1/go.mod h1:vSmSjbyrlKjjsL71193LmzBOKgwePk9DH6uFaWHIInc= -github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= -github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= -github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= -github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= -github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8= -github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= -github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4= -github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA= -github.com/lestrrat-go/strftime v1.0.5 h1:A7H3tT8DhTz8u65w+JRpiBxM4dINQhUXAZnhBa2xeOE= -github.com/lestrrat-go/strftime v1.0.5/go.mod h1:E1nN3pCbtMSu1yjSVeyuRFVm/U0xoR76fd03sz+Qz4g= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= -github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= -github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= -github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= -github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= -github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg= -github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI= -github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= -github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY= -github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI= -github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-oci8 v0.1.1/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= -github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= -github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= -github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= -github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= -github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= -github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= -github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= -github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= -github.com/mistifyio/go-zfs/v3 v3.0.0 h1:J5QK618xRcXnQYZ2GE5FdmpS1ufIrWue+lR/mpe6/14= -github.com/mistifyio/go-zfs/v3 v3.0.0/go.mod h1:CzVgeB0RvF2EGzQnytKVvVSDwmKJXxkOTUGbNrTja/k= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/cli v1.1.2/go.mod h1:6iaV0fGdElS6dPBx0EApTxHrcWvmJphyh2n8YBLPPZ4= -github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= -github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= -github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= -github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= -github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= -github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= -github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= -github.com/moby/moby v20.10.14+incompatible h1:J47P0p+O49F3au8QyE34dE/qXz571kcVmsbx8bvEuS0= -github.com/moby/moby v20.10.14+incompatible/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc= -github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/mountinfo v0.6.1/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= -github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= -github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= -github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= -github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modood/table v0.0.0-20220527013332-8d47e76dad33 h1:T5IbS9C1G2zeHb6eBy6OfIvj5tfQB23kGFpewCJuGDg= -github.com/modood/table v0.0.0-20220527013332-8d47e76dad33/go.mod h1:41qyXVI5QH9/ObyPj27CGCVau5v/njfc3Gjj7yzr0HQ= -github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= -github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= -github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= -github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= -github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= -github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84/go.mod h1:Qnt1q4cjDNQI9bT832ziho5Iw2BhK8o1KwLOwW56VP4= -github.com/opencontainers/image-spec v1.1.0-rc1 h1:lfG+OTa7V8PD3PKvkocSG9KAcA9MANqJn53m31Fvwkc= -github.com/opencontainers/image-spec v1.1.0-rc1/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= -github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= -github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= -github.com/opencontainers/runc v1.1.1/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= -github.com/opencontainers/runc v1.1.4 h1:nRCz/8sKg6K6jgYAFLDlXzPeITBZJyX28DBVhWD+5dg= -github.com/opencontainers/runc v1.1.4/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= -github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= -github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= -github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= -github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= -github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/opencontainers/selinux v1.10.2 h1:NFy2xCsjn7+WspbfZkUd5zyVeisV7VFbPSP96+8/ha4= -github.com/opencontainers/selinux v1.10.2/go.mod h1:cARutUbaUrlRClyvxOICCgKixCs6L05aUsohzA3EkHQ= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f h1:/UDgs8FGMqwnHagNDPGOlts35QkhAZ8by3DR7nMih7M= -github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= -github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= -github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= -github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go= -github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1 h1:oL4IBbcqwhhNWh31bjOX8C/OCy0zs9906d/VUru+bqg= -github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1/go.mod h1:nSbFQvMj97ZyhFRSJYtut+msi4sOY6zJDGCdSc+/rZU= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/proglottis/gpgme v0.1.1/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0= -github.com/proglottis/gpgme v0.1.3 h1:Crxx0oz4LKB3QXc5Ea0J19K/3ICfy3ftr5exgUK1AU0= -github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= -github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo= -github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rubenv/sql-migrate v1.1.1 h1:haR5Hn8hbW9/SpAICrXoZqXnywS7Q5WijwkQENPeNWY= -github.com/rubenv/sql-migrate v1.1.1/go.mod h1:/7TZymwxN8VWumcIxw1jjHEcR1djpdkMHQPT4FWdnbQ= -github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= -github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= -github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= -github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= -github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= -github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= -github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= -github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= -github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= -github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= -github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= -github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= -github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 h1:lIOOHPEbXzO3vnmx2gok1Tfs31Q8GQqKLc8vVqyQq/I= -github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= -github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= -github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= -github.com/sylabs/sif/v2 v2.7.0/go.mod h1:TiyBWsgWeh5yBeQFNuQnvROwswqK7YJT8JA1L53bsXQ= -github.com/sylabs/sif/v2 v2.8.0 h1:FIfWA1fYSFynKD1LJwGbWJ2ib8ylT8XwZl9naLlciPE= -github.com/sylabs/sif/v2 v2.8.0/go.mod h1:LQOdYXC9a8i7BleTKRw9lohi0rTbXkJOeS9u0ebvgyM= -github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= -github.com/tchap/go-patricia v2.3.0+incompatible h1:GkY4dP3cEfEASBPPkWd+AmjYxhmDkqO9/zg7R0lSQRs= -github.com/tchap/go-patricia v2.3.0+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= -github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= -github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= -github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME= -github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= -github.com/vbauerster/mpb/v7 v7.4.1/go.mod h1:Ygg2mV9Vj9sQBWqsK2m2pidcf9H3s6bNKtqd3/M4gBo= -github.com/vbauerster/mpb/v7 v7.5.3 h1:BkGfmb6nMrrBQDFECR/Q7RkKCw7ylMetCb4079CGs4w= -github.com/vbauerster/mpb/v7 v7.5.3/go.mod h1:i+h4QY6lmLvBNK2ah1fSreiw3ajskRlBp9AhY/PnuOE= -github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= -github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk= -github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 h1:p7OofyZ509h8DmPLh8Hn+EIIZm/xYhdZHJ9GnXHdr6U= -github.com/yvasiyarov/gorelic v0.0.7 h1:4DTF1WOM2ZZS/xMOkTFBOcb6XiHu/PKn3rVo6dbewQE= -github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 h1:AsFN8kXcCVkUFHyuzp1FtYbzp1nCO/H6+1uPSGEyPzM= -github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= -github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= -go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= -go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= -go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= -go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= -go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdHZTy8mBTIPo7We18TuO/bak= -go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= -go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= -go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE= -go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= -go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= -go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= -go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= -go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= -go4.org/intern v0.0.0-20211027215823-ae77deb06f29 h1:UXLjNohABv4S58tHmeuIZDO6e3mHpW2Dx33gaNt03LE= -go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 h1:FyBZqvoA/jbNzuAWLQE2kG820zMAkcilx6BMjGbL/E4= -golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 h1:a5Yg6ylndHHYJqIPrdq0AhvR6KTvDTAvgBtaidhEevY= -golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 h1:lxqLZaMad/dJHMFZH0NiNpiEZI/nhgWhe4wgzpE+MuA= -golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ= -golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= -gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= -google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= -google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= -google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= -google.golang.org/api v0.97.0 h1:x/vEL1XDF/2V4xzdNgFPaKHluRESo2aTsL7QzHnBtGQ= -google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce h1:+2ye9vAK4F9F/LCex8dT2cDk0VnTAwUL8uRgX/6nAMU= -google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= -gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= -gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -helm.sh/helm/v3 v3.9.4 h1:TCI1QhJUeLVOdccfdw+vnSEO3Td6gNqibptB04QtExY= -helm.sh/helm/v3 v3.9.4/go.mod h1:3eaWAIqzvlRSD06gR9MMwmp2KBKwlu9av1/1BZpjeWY= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -inet.af/netaddr v0.0.0-20220617031823-097006376321 h1:B4dC8ySKTQXasnjDTMsoCMf1sQG4WsMej0WXaHxunmU= -k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= -k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= -k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs= -k8s.io/api v0.25.4 h1:3YO8J4RtmG7elEgaWMb4HgmpS2CfY1QlaOz9nwB+ZSs= -k8s.io/api v0.25.4/go.mod h1:IG2+RzyPQLllQxnhzD8KQNEu4c4YvyDTpSMztf4A0OQ= -k8s.io/apiextensions-apiserver v0.25.4 h1:7hu9pF+xikxQuQZ7/30z/qxIPZc2J1lFElPtr7f+B6U= -k8s.io/apiextensions-apiserver v0.25.4/go.mod h1:bkSGki5YBoZWdn5pWtNIdGvDrrsRWlmnvl9a+tAw5vQ= -k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= -k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U= -k8s.io/apimachinery v0.25.4 h1:CtXsuaitMESSu339tfhVXhQrPET+EiWnIY1rcurKnAc= -k8s.io/apimachinery v0.25.4/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= -k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= -k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= -k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= -k8s.io/apiserver v0.22.5/go.mod h1:s2WbtgZAkTKt679sYtSudEQrTGWUSQAPe6MupLnlmaQ= -k8s.io/apiserver v0.25.4 h1:/3TwZcgLqX7wUxq7TtXOUqXeBTwXIblVMQdhR5XZ7yo= -k8s.io/apiserver v0.25.4/go.mod h1:rPcm567XxjOnnd7jedDUnGJGmDGAo+cT6H7QHAN+xV0= -k8s.io/cli-runtime v0.25.4 h1:GTSBN7aKBrc2LqpdO30CmHQqJtRmotxV7XsMSP+QZIk= -k8s.io/cli-runtime v0.25.4/go.mod h1:JGOw1CR8v4Mcz6cEKA7bFQe0bPrNn1l5sGAX1/Ke4Eg= -k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= -k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= -k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y= -k8s.io/client-go v0.25.4 h1:3RNRDffAkNU56M/a7gUfXaEzdhZlYhoW8dgViGy5fn8= -k8s.io/client-go v0.25.4/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw= -k8s.io/cluster-bootstrap v0.25.4 h1:m50ICwMsEW13N7Z/tdTmLwELGHt4SJEJaeriPdQRxs0= -k8s.io/cluster-bootstrap v0.25.4/go.mod h1:q6XZzStf/uGcXUZhMHmRB8ael1k+bv8//AajQ6FPJag= -k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= -k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= -k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= -k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI= -k8s.io/component-base v0.25.4 h1:n1bjg9Yt+G1C0WnIDJmg2fo6wbEU1UGMRiQSjmj7hNQ= -k8s.io/component-base v0.25.4/go.mod h1:nnZJU8OP13PJEm6/p5V2ztgX2oyteIaAGKGMYb2L2cY= -k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= -k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= -k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= -k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= -k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= -k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= -k8s.io/kubectl v0.25.4 h1:O3OA1z4V1ZyvxCvScjq0pxAP7ABgznr8UvnVObgI6Dc= -k8s.io/kubectl v0.25.4/go.mod h1:CKMrQ67Bn2YCP26tZStPQGq62zr9pvzEf65A0navm8k= -k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -oras.land/oras-go v1.2.0 h1:yoKosVIbsPoFMqAIFHTnrmOuafHal+J/r+I5bdbVWu4= -oras.land/oras-go v1.2.0/go.mod h1:pFNs7oHp2dYsYMSS82HaX5l4mpnGO7hbpPN6EWH2ltc= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/letsencrypt v0.0.3 h1:H7xDfhkaFFSYEJlKeq38RwX2jYcnTeHuDQyT+mMNMwM= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/cluster-api v1.2.6 h1:wueslUK8LdTH8rOI697WIj1jutbaGY3a24t4XjZvLI8= -sigs.k8s.io/cluster-api v1.2.6/go.mod h1:Ye5gn15u+q6VcE+Se7nBMSo6INm55F+zBcCiWzrjxFc= -sigs.k8s.io/cluster-api/test v1.2.6 h1:QGcLuLsKkjnd6bEAQ9DKVpZJ4PbgvHe8pFV1Nt5dUGA= -sigs.k8s.io/cluster-api/test v1.2.6/go.mod h1:uDGdxAADhTIuQoHoGSYehqMWKQ9sR4/LGE8y2k0Y2LM= -sigs.k8s.io/controller-runtime v0.12.3 h1:FCM8xeY/FI8hoAfh/V4XbbYMY20gElh9yh+A98usMio= -sigs.k8s.io/controller-runtime v0.12.3/go.mod h1:qKsk4WE6zW2Hfj0G4v10EnNB2jMG1C+NTb8h+DwCoU0= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/kind v0.14.0 h1:cNmI3jGBvp7UegEGbC5we8plDtCUmaNRL+bod7JoSCE= -sigs.k8s.io/kind v0.14.0/go.mod h1:UrFRPHG+2a5j0Q7qiR4gtJ4rEyn8TuMQwuOPf+m4oHg= -sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM= -sigs.k8s.io/kustomize/api v0.12.1/go.mod h1:y3JUhimkZkR6sbLNwfJHxvo1TCLwuwm14sCYnkH6S1s= -sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk= -sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ85/nOXac4= -sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/hack/auto-update-version.py b/hack/auto-update-version.py deleted file mode 100755 index f9fa93f9..00000000 --- a/hack/auto-update-version.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python3 -# encoding: utf-8 - -# Copyright 2022 The KubeSphere Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import requests -import re -import json -from natsort import natsorted -import collections - -GITHUB_BASE_URL = "https://api.github.com" -ORG = "kubernetes" -REPO = "kubernetes" -PER_PAGE = 15 - -ARCH_LIST = ["amd64", "arm64"] -K8S_COMPONENTS = ["kubeadm", "kubelet", "kubectl"] - - -def get_releases(org, repo, per_page=30): - try: - response = requests.get("{}/repos/{}/{}/releases?per_page={}".format(GITHUB_BASE_URL, org, repo, per_page)) - except: - print("fetch {}/{} releases failed".format(org, repo)) - else: - return response.json() - - -def get_new_kubernetes_version(current_version): - new_versions = [] - - kubernetes_release = get_releases(org=ORG, repo=REPO, per_page=PER_PAGE) - - for release in kubernetes_release: - tag = release['tag_name'] - res = re.search("^v[0-9]+.[0-9]+.[0-9]+$", tag) - if res and tag not in current_version['kubeadm']['amd64'].keys(): - new_versions.append(tag) - - return new_versions - - -def fetch_kubernetes_sha256(versions): - new_sha256 = {} - - for version in versions: - for binary in K8S_COMPONENTS: - for arch in ARCH_LIST: - response = requests.get( - "https://storage.googleapis.com/kubernetes-release/release/{}/bin/linux/{}/{}.sha256".format( - version, arch, binary)) - if response.status_code == 200: - new_sha256["{}-{}-{}".format(binary, arch, version)] = response.text - - return new_sha256 - - -def version_sort(data): - version_list = natsorted([*data]) - sorted_data = collections.OrderedDict() - - for v in version_list: - sorted_data[v] = data[v] - - return sorted_data - - -def main(): - # get current support versions - with open("version/components.json", "r") as f: - data = json.load(f) - - # get new kubernetes versions - new_versions = get_new_kubernetes_version(current_version=data) - - if len(new_versions) > 0: - # fetch new kubernetes sha256 - new_sha256 = fetch_kubernetes_sha256(new_versions) - - if new_sha256: - for k, v in new_sha256.items(): - info = k.split('-') - data[info[0]][info[1]][info[2]] = v - - for binary in K8S_COMPONENTS: - for arch in ARCH_LIST: - data[binary][arch] = version_sort(data[binary][arch]) - - print(new_versions) - # update components.json - with open("version/components.json", 'w') as f: - json.dump(data, f, indent=4, ensure_ascii=False) - - # set new version to tmp file - with open("version.tmp", 'w') as f: - f.write("\n".join(new_versions)) - - -if __name__ == '__main__': - main() diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt deleted file mode 100644 index 62802d18..00000000 --- a/hack/boilerplate.go.txt +++ /dev/null @@ -1,15 +0,0 @@ -/* -Copyright 2022 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ \ No newline at end of file diff --git a/hack/ensure-golangci-lint.sh b/hack/ensure-golangci-lint.sh deleted file mode 100755 index 8c2729eb..00000000 --- a/hack/ensure-golangci-lint.sh +++ /dev/null @@ -1,422 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2021 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# NOTE: This script is copied from from https://raw.githubusercontent.com/golangci/golangci-lint/main/install.sh. - -set -e - -usage() { - this=$1 - cat </dev/null -} -echoerr() { - echo "$@" 1>&2 -} -log_prefix() { - echo "$0" -} -_logp=6 -log_set_priority() { - _logp="$1" -} -log_priority() { - if test -z "$1"; then - echo "$_logp" - return - fi - [ "$1" -le "$_logp" ] -} -log_tag() { - case $1 in - 0) echo "emerg" ;; - 1) echo "alert" ;; - 2) echo "crit" ;; - 3) echo "err" ;; - 4) echo "warning" ;; - 5) echo "notice" ;; - 6) echo "info" ;; - 7) echo "debug" ;; - *) echo "$1" ;; - esac -} -log_debug() { - log_priority 7 || return 0 - echoerr "$(log_prefix)" "$(log_tag 7)" "$@" -} -log_info() { - log_priority 6 || return 0 - echoerr "$(log_prefix)" "$(log_tag 6)" "$@" -} -log_err() { - log_priority 3 || return 0 - echoerr "$(log_prefix)" "$(log_tag 3)" "$@" -} -log_crit() { - log_priority 2 || return 0 - echoerr "$(log_prefix)" "$(log_tag 2)" "$@" -} -uname_os() { - os=$(uname -s | tr '[:upper:]' '[:lower:]') - case "$os" in - cygwin_nt*) os="windows" ;; - mingw*) os="windows" ;; - msys_nt*) os="windows" ;; - esac - echo "$os" -} -uname_arch() { - arch=$(uname -m) - case $arch in - x86_64) arch="amd64" ;; - x86) arch="386" ;; - i686) arch="386" ;; - i386) arch="386" ;; - aarch64) arch="arm64" ;; - armv5*) arch="armv5" ;; - armv6*) arch="armv6" ;; - armv7*) arch="armv7" ;; - esac - echo ${arch} -} -uname_os_check() { - os=$(uname_os) - case "$os" in - darwin) return 0 ;; - dragonfly) return 0 ;; - freebsd) return 0 ;; - linux) return 0 ;; - android) return 0 ;; - nacl) return 0 ;; - netbsd) return 0 ;; - openbsd) return 0 ;; - plan9) return 0 ;; - solaris) return 0 ;; - windows) return 0 ;; - esac - log_crit "uname_os_check '$(uname -s)' got converted to '$os' which is not a GOOS value. Please file bug at https://github.com/client9/shlib" - return 1 -} -uname_arch_check() { - arch=$(uname_arch) - case "$arch" in - 386) return 0 ;; - amd64) return 0 ;; - arm64) return 0 ;; - armv5) return 0 ;; - armv6) return 0 ;; - armv7) return 0 ;; - ppc64) return 0 ;; - ppc64le) return 0 ;; - mips) return 0 ;; - mipsle) return 0 ;; - mips64) return 0 ;; - mips64le) return 0 ;; - s390x) return 0 ;; - amd64p32) return 0 ;; - esac - log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value. Please file bug report at https://github.com/client9/shlib" - return 1 -} -untar() { - tarball=$1 - case "${tarball}" in - *.tar.gz | *.tgz) tar --no-same-owner -xzf "${tarball}" ;; - *.tar) tar --no-same-owner -xf "${tarball}" ;; - *.zip) unzip "${tarball}" ;; - *) - log_err "untar unknown archive format for ${tarball}" - return 1 - ;; - esac -} -http_download_curl() { - local_file=$1 - source_url=$2 - header=$3 - if [ -z "$header" ]; then - code=$(curl -w '%{http_code}' -sL -o "$local_file" "$source_url") - else - code=$(curl -w '%{http_code}' -sL -H "$header" -o "$local_file" "$source_url") - fi - if [ "$code" != "200" ]; then - log_debug "http_download_curl received HTTP status $code" - return 1 - fi - return 0 -} -http_download_wget() { - local_file=$1 - source_url=$2 - header=$3 - if [ -z "$header" ]; then - wget -q -O "$local_file" "$source_url" - else - wget -q --header "$header" -O "$local_file" "$source_url" - fi -} -http_download() { - log_debug "http_download $2" - if is_command curl; then - http_download_curl "$@" - return - elif is_command wget; then - http_download_wget "$@" - return - fi - log_crit "http_download unable to find wget or curl" - return 1 -} -http_copy() { - tmp=$(mktemp) - http_download "${tmp}" "$1" "$2" || return 1 - body=$(cat "$tmp") - rm -f "${tmp}" - echo "$body" -} -github_release() { - owner_repo=$1 - version=$2 - test -z "$version" && version="latest" - giturl="https://github.com/${owner_repo}/releases/${version}" - json=$(http_copy "$giturl" "Accept:application/json") - test -z "$json" && return 1 - version=$(echo "$json" | tr -s '\n' ' ' | sed 's/.*"tag_name":"//' | sed 's/".*//') - test -z "$version" && return 1 - echo "$version" -} -hash_sha256() { - TARGET=${1:-/dev/stdin} - if is_command gsha256sum; then - hash=$(gsha256sum "$TARGET") || return 1 - echo "$hash" | cut -d ' ' -f 1 - elif is_command sha256sum; then - hash=$(sha256sum "$TARGET") || return 1 - echo "$hash" | cut -d ' ' -f 1 - elif is_command shasum; then - hash=$(shasum -a 256 "$TARGET" 2>/dev/null) || return 1 - echo "$hash" | cut -d ' ' -f 1 - elif is_command openssl; then - hash=$(openssl -dst openssl dgst -sha256 "$TARGET") || return 1 - echo "$hash" | cut -d ' ' -f a - else - log_crit "hash_sha256 unable to find command to compute sha-256 hash" - return 1 - fi -} -hash_sha256_verify() { - TARGET=$1 - checksums=$2 - if [ -z "$checksums" ]; then - log_err "hash_sha256_verify checksum file not specified in arg2" - return 1 - fi - BASENAME=${TARGET##*/} - want=$(grep "${BASENAME}" "${checksums}" 2>/dev/null | tr '\t' ' ' | cut -d ' ' -f 1) - if [ -z "$want" ]; then - log_err "hash_sha256_verify unable to find checksum for '${TARGET}' in '${checksums}'" - return 1 - fi - got=$(hash_sha256 "$TARGET") - if [ "$want" != "$got" ]; then - log_err "hash_sha256_verify checksum for '$TARGET' did not verify ${want} vs $got" - return 1 - fi -} -cat /dev/null < "${file}" && echo -e "\n\nThe hash info have saved to file ${file}.\n\n" diff --git a/hack/gen-repository-iso/dockerfile.almalinux90 b/hack/gen-repository-iso/dockerfile.almalinux90 deleted file mode 100644 index a90c8b70..00000000 --- a/hack/gen-repository-iso/dockerfile.almalinux90 +++ /dev/null @@ -1,21 +0,0 @@ -FROM almalinux:9.0 as almalinux90 -ARG TARGETARCH -ARG BUILD_TOOLS="dnf-plugins-core createrepo mkisofs epel-release" -ARG DIR=almalinux-9.0-${TARGETARCH}-rpms -ARG PKGS=.common[],.rpms[],.almalinux[],.almalinux90[] - -RUN dnf install -q -y ${BUILD_TOOLS} \ - && dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo \ - && dnf makecache - -WORKDIR package -COPY packages.yaml . -COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq -RUN yq eval ${PKGS} packages.yaml | sed '/^ceph-common$/d' > packages.list - -RUN sort -u packages.list | xargs dnf download --resolve --alldeps --downloaddir=${DIR} \ - && createrepo -d ${DIR} \ - && mkisofs -r -o ${DIR}.iso ${DIR} - -FROM scratch -COPY --from=almalinux90 /package/*.iso / diff --git a/hack/gen-repository-iso/dockerfile.centos7 b/hack/gen-repository-iso/dockerfile.centos7 deleted file mode 100644 index e0ed0cfc..00000000 --- a/hack/gen-repository-iso/dockerfile.centos7 +++ /dev/null @@ -1,22 +0,0 @@ -FROM centos:7 as centos7 -ARG TARGETARCH -ENV OS=centos -ENV OS_VERSION=7 -ARG BUILD_TOOLS="yum-utils createrepo mkisofs epel-release" -ARG DIR=${OS}${OS_VERSION}-${TARGETARCH}-rpms - -RUN yum install -q -y ${BUILD_TOOLS} \ - && yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo \ - && yum makecache - -WORKDIR package -COPY packages.yaml . -COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq -RUN yq eval ".common[],.rpms[],.${OS}[],.${OS}${OS_VERSION}[]" packages.yaml > packages.list - -RUN sort -u packages.list | xargs repotrack -p ${DIR} \ - && createrepo -d ${DIR} \ - && mkisofs -r -o ${DIR}.iso ${DIR} - -FROM scratch -COPY --from=centos7 /package/*.iso / diff --git a/hack/gen-repository-iso/dockerfile.debian10 b/hack/gen-repository-iso/dockerfile.debian10 deleted file mode 100644 index 635a124d..00000000 --- a/hack/gen-repository-iso/dockerfile.debian10 +++ /dev/null @@ -1,38 +0,0 @@ -FROM debian:10 as debian10 -ARG TARGETARCH -ARG OS_RELEASE=buster -ARG OS_VERSION=10 -ARG DIR=debian-10-${TARGETARCH}-debs -ARG PKGS=.common[],.debs[],.debian[],.debian10[] -ARG BUILD_TOOLS="apt-transport-https software-properties-common ca-certificates curl wget gnupg dpkg-dev genisoimage dirmngr" -ENV DEBIAN_FRONTEND=noninteractive - -# dump system package list -RUN dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d ':' -f1 > packages.list -RUN ARCH=$(dpkg --print-architecture) \ - && apt update -qq \ - && apt install -y --no-install-recommends $BUILD_TOOLS \ - && if [ "$TARGETARCH" = "amd64" ]; then \ - curl -fsSL https://download.gluster.org/pub/gluster/glusterfs/7/rsa.pub | apt-key add - ; \ - echo deb https://download.gluster.org/pub/gluster/glusterfs/7/LATEST/Debian/${OS_VERSION}/amd64/apt ${OS_RELEASE} main > /etc/apt/sources.list.d/gluster.list ; \ - fi \ - && curl -fsSL "https://download.docker.com/linux/debian/gpg" | apt-key add -qq - \ - && echo "deb [arch=$TARGETARCH] https://download.docker.com/linux/debian ${OS_RELEASE} stable" > /etc/apt/sources.list.d/docker.list \ - && apt update -qq - -WORKDIR /package -COPY packages.yaml . - -COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq -RUN yq eval "${PKGS}" packages.yaml >> packages.list \ - && sort -u packages.list | xargs apt-get install --yes --reinstall --print-uris | awk -F "'" '{print $2}' | grep -v '^$' | sort -u > packages.urls - -RUN mkdir -p ${DIR} \ - && wget -q -x -P ${DIR} -i packages.urls \ - && cd ${DIR} \ - && dpkg-scanpackages ./ /dev/null | gzip -9c > ./Packages.gz - -RUN genisoimage -r -o ${DIR}.iso ${DIR} - -FROM scratch -COPY --from=debian10 /package/*.iso / diff --git a/hack/gen-repository-iso/dockerfile.debian11 b/hack/gen-repository-iso/dockerfile.debian11 deleted file mode 100644 index f99dd95f..00000000 --- a/hack/gen-repository-iso/dockerfile.debian11 +++ /dev/null @@ -1,41 +0,0 @@ -FROM debian:11.6 as debian11 -ARG TARGETARCH -ARG OS_RELEASE=bullseye -ARG OS_VERSION=11 -ARG DIR=debian-11-${TARGETARCH}-debs -ARG PKGS=.common[],.debs[],.debian[],.debian11[] -ARG BUILD_TOOLS="apt-transport-https software-properties-common ca-certificates curl wget gnupg dpkg-dev genisoimage dirmngr" -ENV DEBIAN_FRONTEND=noninteractive - -# dump system package list -RUN dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d ':' -f1 > packages.list -RUN ARCH=$(dpkg --print-architecture) \ - && apt update -qq \ - && apt install -y --no-install-recommends $BUILD_TOOLS \ - && if [ "$TARGETARCH" = "amd64" ]; then \ - curl -fsSL https://download.gluster.org/pub/gluster/glusterfs/7/rsa.pub | apt-key add - ; \ - echo deb https://download.gluster.org/pub/gluster/glusterfs/7/LATEST/Debian/${OS_VERSION}/amd64/apt ${OS_RELEASE} main > /etc/apt/sources.list.d/gluster.list ; \ - fi \ - && curl -fsSL "https://download.docker.com/linux/debian/gpg" | apt-key add -qq - \ - && echo "deb [arch=$TARGETARCH] https://download.docker.com/linux/debian ${OS_RELEASE} stable" > /etc/apt/sources.list.d/docker.list \ - && apt update -qq \ - && apt upgrade -y -qq - -WORKDIR /package -COPY packages.yaml . - -COPY --from=mikefarah/yq:4.30.8 /usr/bin/yq /usr/bin/yq -RUN yq eval "${PKGS}" packages.yaml >> packages.list \ - && sort -u packages.list | xargs apt-get install --yes --reinstall --print-uris | awk -F "'" '{print $2}' | grep -v '^$' | sort -u > packages.urls - -RUN cat packages.urls - -RUN mkdir -p ${DIR} \ - && wget -q -x -P ${DIR} -i packages.urls \ - && cd ${DIR} \ - && dpkg-scanpackages ./ /dev/null | gzip -9c > ./Packages.gz - -RUN genisoimage -r -o ${DIR}.iso ${DIR} - -FROM scratch -COPY --from=debian11 /package/*.iso / diff --git a/hack/gen-repository-iso/dockerfile.ubuntu1604 b/hack/gen-repository-iso/dockerfile.ubuntu1604 deleted file mode 100644 index 71969819..00000000 --- a/hack/gen-repository-iso/dockerfile.ubuntu1604 +++ /dev/null @@ -1,33 +0,0 @@ -FROM ubuntu:16.04 as ubuntu1604 -ARG TARGETARCH -ARG OS_RELEASE=xenial -ARG DIR=ubuntu-16.04-${TARGETARCH}-debs -ARG PKGS=.common[],.debs[],.ubuntu[],.ubuntu1604[] -ARG BUILD_TOOLS="apt-transport-https software-properties-common ca-certificates curl wget gnupg dpkg-dev genisoimage" -ENV DEBIAN_FRONTEND=noninteractive - -# dump system package list -RUN dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d ':' -f1 > packages.list -RUN apt update -qq \ - && apt install -y --no-install-recommends $BUILD_TOOLS \ - && add-apt-repository ppa:gluster/glusterfs-7 -y \ - && curl -fsSL "https://download.docker.com/linux/ubuntu/gpg" | apt-key add -qq - \ - && echo "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu ${OS_RELEASE} stable" > /etc/apt/sources.list.d/docker.list\ - && apt update -qq - -WORKDIR /package -COPY packages.yaml . - -COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq -RUN yq eval "${PKGS}" packages.yaml >> packages.list \ - && sort -u packages.list | xargs apt-get install --yes --reinstall --print-uris | awk -F "'" '{print $2}' | grep -v '^$' | sort -u > packages.urls - -RUN mkdir -p ${DIR} \ - && wget -q -x -P ${DIR} -i packages.urls \ - && cd ${DIR} \ - && dpkg-scanpackages ./ /dev/null | gzip -9c > ./Packages.gz - -RUN genisoimage -r -o ${DIR}.iso ${DIR} - -FROM scratch -COPY --from=ubuntu1604 /package/*.iso / diff --git a/hack/gen-repository-iso/dockerfile.ubuntu1804 b/hack/gen-repository-iso/dockerfile.ubuntu1804 deleted file mode 100644 index f9852d8d..00000000 --- a/hack/gen-repository-iso/dockerfile.ubuntu1804 +++ /dev/null @@ -1,34 +0,0 @@ -FROM ubuntu:18.04 as ubuntu1804 -ARG TARGETARCH -ARG OS_RELEASE=bionic -ARG DIR=ubuntu-18.04-${TARGETARCH}-debs -ARG PKGS=.common[],.debs[],.ubuntu[],.ubuntu1804[] -ARG BUILD_TOOLS="apt-transport-https software-properties-common ca-certificates curl wget gnupg dpkg-dev genisoimage" -ENV DEBIAN_FRONTEND=noninteractive - -# dump system package list -RUN dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d ':' -f1 > packages.list -RUN apt update -qq \ - && apt install -y --no-install-recommends $BUILD_TOOLS \ - && add-apt-repository ppa:gluster/glusterfs-7 -y \ - && curl -fsSL "https://download.docker.com/linux/ubuntu/gpg" | apt-key add -qq - \ - && echo "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu ${OS_RELEASE} stable" > /etc/apt/sources.list.d/docker.list\ - && apt update -qq - -WORKDIR /package -COPY packages.yaml . - -COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq -RUN yq eval "${PKGS}" packages.yaml >> packages.list \ - && dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d ':' -f1 >> packages.list \ - && sort -u packages.list | xargs apt-get install --yes --reinstall --print-uris | awk -F "'" '{print $2}' | grep -v '^$' | sort -u > packages.urls - -RUN mkdir -p ${DIR} \ - && wget -q -x -P ${DIR} -i packages.urls \ - && cd ${DIR} \ - && dpkg-scanpackages ./ /dev/null | gzip -9c > ./Packages.gz - -RUN genisoimage -r -o ${DIR}.iso ${DIR} - -FROM scratch -COPY --from=ubuntu1804 /package/*.iso / diff --git a/hack/gen-repository-iso/dockerfile.ubuntu2004 b/hack/gen-repository-iso/dockerfile.ubuntu2004 deleted file mode 100644 index 9cb4f0b2..00000000 --- a/hack/gen-repository-iso/dockerfile.ubuntu2004 +++ /dev/null @@ -1,33 +0,0 @@ -FROM ubuntu:20.04 as ubuntu2004 -ARG TARGETARCH -ARG OS_RELEASE=focal -ARG DIR=ubuntu-20.04-${TARGETARCH}-debs -ARG PKGS=.common[],.debs[],.ubuntu[],.ubuntu2004[] -ARG BUILD_TOOLS="apt-transport-https software-properties-common ca-certificates curl wget gnupg dpkg-dev genisoimage" -ENV DEBIAN_FRONTEND=noninteractive - -# dump system package list -RUN dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d ':' -f1 > packages.list -RUN apt update -qq \ - && apt install -y --no-install-recommends $BUILD_TOOLS \ - && add-apt-repository ppa:gluster/glusterfs-7 -y \ - && curl -fsSL "https://download.docker.com/linux/ubuntu/gpg" | apt-key add -qq - \ - && echo "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu ${OS_RELEASE} stable" > /etc/apt/sources.list.d/docker.list\ - && apt update -qq - -WORKDIR /package -COPY packages.yaml . - -COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq -RUN yq eval "${PKGS}" packages.yaml >> packages.list \ - && sort -u packages.list | xargs apt-get install --yes --reinstall --print-uris | awk -F "'" '{print $2}' | grep -v '^$' | sort -u > packages.urls - -RUN mkdir -p ${DIR} \ - && wget -q -x -P ${DIR} -i packages.urls \ - && cd ${DIR} \ - && dpkg-scanpackages ./ /dev/null | gzip -9c > ./Packages.gz - -RUN genisoimage -r -o ${DIR}.iso ${DIR} - -FROM scratch -COPY --from=ubuntu2004 /package/*.iso / diff --git a/hack/gen-repository-iso/dockerfile.ubuntu2204 b/hack/gen-repository-iso/dockerfile.ubuntu2204 deleted file mode 100644 index 7a92912a..00000000 --- a/hack/gen-repository-iso/dockerfile.ubuntu2204 +++ /dev/null @@ -1,33 +0,0 @@ -FROM ubuntu:22.04 as ubuntu2204 -ARG TARGETARCH -ARG OS_RELEASE=jammy -ARG DIR=ubuntu-22.04-${TARGETARCH}-debs -ARG PKGS=.common[],.debs[],.ubuntu[],.ubuntu2204[] -ARG BUILD_TOOLS="apt-transport-https software-properties-common ca-certificates curl wget gnupg dpkg-dev genisoimage" -ENV DEBIAN_FRONTEND=noninteractive - -# dump system package list -RUN dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d ':' -f1 > packages.list -RUN apt update -qq \ - && apt install -y --no-install-recommends $BUILD_TOOLS \ - #&& add-apt-repository ppa:gluster/glusterfs-7 -y \ - && curl -fsSL "https://download.docker.com/linux/ubuntu/gpg" | apt-key add -qq - \ - && echo "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu ${OS_RELEASE} stable" > /etc/apt/sources.list.d/docker.list\ - && apt update -qq - -WORKDIR /package -COPY packages.yaml . - -COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq -RUN yq eval "${PKGS}" packages.yaml >> packages.list \ - && sort -u packages.list | xargs apt-get install --yes --reinstall --print-uris | awk -F "'" '{print $2}' | grep -v '^$' | sort -u > packages.urls - -RUN mkdir -p ${DIR} \ - && wget -q -x -P ${DIR} -i packages.urls \ - && cd ${DIR} \ - && dpkg-scanpackages ./ /dev/null | gzip -9c > ./Packages.gz - -RUN genisoimage -r -o ${DIR}.iso ${DIR} - -FROM scratch -COPY --from=ubuntu2204 /package/*.iso / diff --git a/hack/gen-repository-iso/download-pkgs.sh b/hack/gen-repository-iso/download-pkgs.sh deleted file mode 100644 index ee0afa35..00000000 --- a/hack/gen-repository-iso/download-pkgs.sh +++ /dev/null @@ -1,7 +0,0 @@ -#! /bin/sh - -for p in ${PACKAGES} ; do - echo "\n Download $p ... \n" - sudo apt-get download $p 2>>errors.txt - for i in $(apt-cache depends $p | grep -E 'Depends|Recommends|Suggests' | cut -d ':' -f 2,3 | sed -e s/' '/''/); do sudo apt-get download $i 2>>errors.txt; done -done diff --git a/hack/gen-repository-iso/packages.yaml b/hack/gen-repository-iso/packages.yaml deleted file mode 100644 index 65d89a3e..00000000 --- a/hack/gen-repository-iso/packages.yaml +++ /dev/null @@ -1,88 +0,0 @@ ---- -common: - - curl - - ceph-common - - net-tools - - lvm2 - - telnet - - tcpdump - - socat - - openssl - - chrony - - conntrack - - curl - - ipvsadm - - ipset - - psmisc - - bash-completion - - ebtables - - haproxy - - keepalived -rpms: - - nfs-utils - - yum-utils - - bind-utils - - glusterfs-fuse - - lz4 - - nss - - nss-sysinit - - nss-tools - - conntrack-tools -debs: - - apt-transport-https - - ca-certificates - - dnsutils - - git - - glusterfs-client - - gnupg-agent - - nfs-common - - openssh-server - - software-properties-common - - sudo - -centos: - - containerd.io - -centos7: - - libselinux-python - - docker-ce-20.10.8 - - docker-ce-cli-20.10.8 - -debian: - - containerd.io - -debian10: - - docker-ce=5:20.10.8~3-0~debian-buster - - docker-ce-cli=5:20.10.8~3-0~debian-buster - -debian11: - - docker-ce=5:20.10.8~3-0~debian-bullseye - - docker-ce-cli=5:20.10.8~3-0~debian-bullseye - -ubuntu: - - containerd.io - -ubuntu1604: - - docker-ce=5:20.10.8~3-0~ubuntu-xenial - - docker-ce-cli=5:20.10.8~3-0~ubuntu-xenial - -ubuntu1804: - - docker-ce=5:20.10.8~3-0~ubuntu-bionic - - docker-ce-cli=5:20.10.8~3-0~ubuntu-bionic - -ubuntu2004: - - docker-ce=5:20.10.8~3-0~ubuntu-focal - - docker-ce-cli=5:20.10.8~3-0~ubuntu-focal - -# The minimum version of docker-ce on ubuntu 2204 is 20.10.13 -ubuntu2204: - - docker-ce=5:20.10.13~3-0~ubuntu-jammy - - docker-ce-cli=5:20.10.13~3-0~ubuntu-jammy - -almalinux: - - containerd.io - - docker-compose-plugin - -almalinux90: - - docker-ce-20.10.17 - - docker-ce-cli-20.10.17 diff --git a/hack/sync-components.sh b/hack/sync-components.sh deleted file mode 100755 index d6585e89..00000000 --- a/hack/sync-components.sh +++ /dev/null @@ -1,340 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2022 The KubeSphere Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -##################################################################### -# -# Usage: -# Specify the component version through environment variables. -# -# For example: -# -# KUBERNETES_VERSION=v1.25.3 bash hack/sync-components.sh -# -#################################################################### - -set -e - -KUBERNETES_VERSION=${KUBERNETES_VERSION} -NODE_LOCAL_DNS_VERSION=${NODE_LOCAL_DNS_VERSION} -COREDNS_VERSION=${COREDNS_VERSION} -CALICO_VERSION=${CALICO_VERSION} -KUBE_OVN_VERSION=${KUBE_OVN_VERSION} -CILIUM_VERSION=${CILIUM_VERSION} -OPENEBS_VERSION=${OPENEBS_VERSION} -KUBEVIP_VERSION=${KUBEVIP_VERSION} -HAPROXY_VERSION=${HAPROXY_VERSION} -HELM_VERSION=${HELM_VERSION} -CNI_VERSION=${CNI_VERSION} -ETCD_VERSION=${ETCD_VERSION} -CRICTL_VERSION=${CRICTL_VERSION} -K3S_VERSION=${K3S_VERSION} -CONTAINERD_VERSION=${CONTAINERD_VERSION} -RUNC_VERSION=${RUNC_VERSION} -COMPOSE_VERSION=${COMPOSE_VERSION} -CALICO_VERSION=${CALICO_VERSION} - -# qsctl -QSCTL_ACCESS_KEY_ID=${QSCTL_ACCESS_KEY_ID} -QSCTL_SECRET_ACCESS_KEY=${QSCTL_SECRET_ACCESS_KEY} - -# docker.io -DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME} -DOCKERHUB_PASSWORD=${DOCKERHUB_PASSWORD} - -# registry.cn-beijing.aliyuncs.com -ALIYUNCS_USERNAME=${ALIYUNCS_USERNAME} -ALIYUNCS_PASSWORD=${ALIYUNCS_PASSWORD} - -DOCKERHUB_NAMESPACE="kubesphere" -ALIYUNCS_NAMESPACE="kubesphereio" - -BINARIES=("kubeadm" "kubelet" "kubectl") -ARCHS=("amd64" "arm64") - -# Generate qsctl config -if [ $QSCTL_ACCESS_KEY_ID ] && [ $QSCTL_SECRET_ACCESS_KEY ];then - echo "access_key_id: $QSCTL_ACCESS_KEY_ID" > qsctl-config.yaml - echo "secret_access_key: $QSCTL_SECRET_ACCESS_KEY" >> qsctl-config.yaml -fi - -# Login docker.io -if [ $DOCKERHUB_USERNAME ] && [ $DOCKERHUB_PASSWORD ];then - skopeo login docker.io -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD -fi - -# Login registry.cn-beijing.aliyuncs.com -if [ $ALIYUNCS_USERNAME ] && [ $ALIYUNCS_PASSWORD ];then - skopeo login registry.cn-beijing.aliyuncs.com -u $ALIYUNCS_USERNAME -p $ALIYUNCS_PASSWORD -fi - -# Sync Kubernetes Binaries and Images -if [ $KUBERNETES_VERSION ]; then - for arch in ${ARCHS[@]} - do - mkdir -p binaries/kube/$KUBERNETES_VERSION/$arch - for binary in ${BINARIES[@]} - do - echo "Synchronizing $binary-$arch" - - curl -L -o binaries/kube/$KUBERNETES_VERSION/$arch/$binary \ - https://storage.googleapis.com/kubernetes-release/release/$KUBERNETES_VERSION/bin/linux/$arch/$binary - - qsctl cp binaries/kube/$KUBERNETES_VERSION/$arch/$binary \ - qs://kubernetes-release/release/$KUBERNETES_VERSION/bin/linux/$arch/$binary \ - -c qsctl-config.yaml - done - done - - chmod +x binaries/kube/$KUBERNETES_VERSION/amd64/kubeadm - binaries/kube/$KUBERNETES_VERSION/amd64/kubeadm config images list | xargs -I {} skopeo sync --src docker --dest docker {} docker.io/$DOCKERHUB_NAMESPACE/${image##} --all - binaries/kube/$KUBERNETES_VERSION/amd64/kubeadm config images list | xargs -I {} skopeo sync --src docker --dest docker {} registry.cn-beijing.aliyuncs.com/$ALIYUNCS_NAMESPACE/${image##} --all - - rm -rf binaries -fi - -# Sync Helm Binary -if [ $HELM_VERSION ]; then - for arch in ${ARCHS[@]} - do - mkdir -p binaries/helm/$HELM_VERSION/$arch - echo "Synchronizing helm-$arch" - - curl -L -o binaries/helm/$HELM_VERSION/$arch/helm-$HELM_VERSION-linux-$arch.tar.gz \ - https://get.helm.sh/helm-$HELM_VERSION-linux-$arch.tar.gz - - tar -zxf binaries/helm/$HELM_VERSION/$arch/helm-$HELM_VERSION-linux-$arch.tar.gz -C binaries/helm/$HELM_VERSION/$arch - - qsctl cp $KUBERNETES_VERSION/$arch/linux-$arch/helm \ - qs://kubernetes-helm/linux-$arch/$HELM_VERSION/helm \ - -c qsctl-config.yaml - - qsctl cp binaries/helm/$HELM_VERSION/$arch/helm-$HELM_VERSION-linux-$arch.tar.gz \ - qs://kubernetes-helm/linux-$arch/$HELM_VERSION/helm-$HELM_VERSION-linux-$arch.tar.gz \ - -c qsctl-config.yaml - done - - rm -rf binaries -fi - -# Sync ETCD Binary -if [ $ETCD_VERSION ]; then - for arch in ${ARCHS[@]} - do - mkdir -p binaries/etcd/$ETCD_VERSION/$arch - echo "Synchronizing etcd-$arch" - - curl -L -o binaries/etcd/$ETCD_VERSION/$arch/etcd-$ETCD_VERSION-linux-$arch.tar.gz \ - https://github.com/coreos/etcd/releases/download/$ETCD_VERSION/etcd-$ETCD_VERSION-linux-$arch.tar.gz - - qsctl cp binaries/etcd/$ETCD_VERSION/$arch/etcd-$ETCD_VERSION-linux-$arch.tar.gz \ - qs://kubernetes-release/etcd/release/download/$ETCD_VERSION/etcd-$ETCD_VERSION-linux-$arch.tar.gz \ - -c qsctl-config.yaml - done - - rm -rf binaries -fi - -# Sync CNI Binary -if [ $CNI_VERSION ]; then - for arch in ${ARCHS[@]} - do - mkdir -p binaries/cni/$CNI_VERSION/$arch - echo "Synchronizing cni-$arch" - - curl -L -o binaries/cni/$CNI_VERSION/$arch/cni-plugins-linux-$arch-$CNI_VERSION.tgz \ - https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION/cni-plugins-linux-$arch-$CNI_VERSION.tgz - - qsctl cp binaries/cni/$CNI_VERSION/$arch/cni-plugins-linux-$arch-$CNI_VERSION.tgz \ - qs://containernetworking/plugins/releases/download/$CNI_VERSION/cni-plugins-linux-$arch-$CNI_VERSION.tgz \ - -c qsctl-config.yaml - done - - rm -rf binaries -fi - -# Sync CALICOCTL Binary -if [ $CALICO_VERSION ]; then - for arch in ${ARCHS[@]} - do - mkdir -p binaries/calicoctl/$CALICO_VERSION/$arch - echo "Synchronizing calicoctl-$arch" - - curl -L -o binaries/calicoctl/$CALICO_VERSION/$arch/calicoctl-linux-$arch \ - https://github.com/projectcalico/calico/releases/download/$CALICO_VERSION/calicoctl-linux-$arch - - qsctl cp binaries/calicoctl/$CALICO_VERSION/$arch/calicoctl-linux-$arch \ - qs://kubernetes-release/projectcalico/calico/releases/download/$CALICO_VERSION/calicoctl-linux-$arch \ - -c qsctl-config.yaml - done - - rm -rf binaries -fi - -# Sync crictl Binary -if [ $CRICTL_VERSION ]; then - echo "access_key_id: $ACCESS_KEY_ID" > qsctl-config.yaml - echo "secret_access_key: $SECRET_ACCESS_KEY" >> qsctl-config.yaml - - for arch in ${ARCHS[@]} - do - mkdir -p binaries/crictl/$CRICTL_VERSION/$arch - echo "Synchronizing crictl-$arch" - - curl -L -o binaries/crictl/$CRICTL_VERSION/$arch/crictl-$CRICTL_VERSION-linux-$arch.tar.gz \ - https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-$CRICTL_VERSION-linux-$arch.tar.gz - - qsctl cp binaries/crictl/$CRICTL_VERSION/$arch/crictl-$CRICTL_VERSION-linux-$arch.tar.gz \ - qs://kubernetes-release/cri-tools/releases/download/$CRICTL_VERSION/crictl-$CRICTL_VERSION-linux-$arch.tar.gz \ - -c qsctl-config.yaml - done - - rm -rf binaries -fi - -# Sync k3s Binary -if [ $K3S_VERSION ]; then - for arch in ${ARCHS[@]} - do - mkdir -p binaries/k3s/$K3S_VERSION/$arch - echo "Synchronizing k3s-$arch" - if [ $arch != "amd64" ]; then - curl -L -o binaries/k3s/$K3S_VERSION/$arch/k3s \ - https://github.com/rancher/k3s/releases/download/$K3S_VERSION+k3s1/k3s-$arch - else - curl -L -o binaries/k3s/$K3S_VERSION/$arch/k3s \ - https://github.com/rancher/k3s/releases/download/$K3S_VERSION+k3s1/k3s - fi - qsctl cp binaries/k3s/$K3S_VERSION/$arch/k3s \ - qs://kubernetes-release/k3s/releases/download/$K3S_VERSION+k3s1/linux/$arch/k3s \ - -c qsctl-config.yaml - done - - rm -rf binaries -fi - -# Sync containerd Binary -if [ $CONTAINERD_VERSION ]; then - for arch in ${ARCHS[@]} - do - mkdir -p binaries/containerd/$CONTAINERD_VERSION/$arch - echo "Synchronizing containerd-$arch" - - curl -L -o binaries/containerd/$CONTAINERD_VERSION/$arch/containerd-$CONTAINERD_VERSION-linux-$arch.tar.gz \ - https://github.com/containerd/containerd/releases/download/v$CONTAINERD_VERSION/containerd-$CONTAINERD_VERSION-linux-$arch.tar.gz - - qsctl cp binaries/containerd/$CONTAINERD_VERSION/$arch/containerd-$CONTAINERD_VERSION-linux-$arch.tar.gz \ - qs://kubernetes-releas/containerd/containerd/releases/download/v$CONTAINERD_VERSION/containerd-$CONTAINERD_VERSION-linux-$arch.tar.gz \ - -c qsctl-config.yaml - done - - rm -rf binaries -fi - -# Sync runc Binary -if [ $RUNC_VERSION ]; then - for arch in ${ARCHS[@]} - do - mkdir -p binaries/runc/$RUNC_VERSION/$arch - echo "Synchronizing runc-$arch" - - curl -L -o binaries/runc/$RUNC_VERSION/$arch/runc.$arch \ - https://github.com/opencontainers/runc/releases/download/$RUNC_VERSION/runc.$arch - - qsctl cp binaries/runc/$RUNC_VERSION/$arch/runc.$arch \ - qs://kubernetes-release/opencontainers/runc/releases/download/$RUNC_VERSION/runc.$arch \ - -c qsctl-config.yaml - done - - rm -rf binaries -fi - -# Sync docker-compose Binary -if [ $RUNC_VERSION ]; then - for arch in ${ARCHS[@]} - do - mkdir -p binaries/compose/$COMPOSE_VERSION/$arch - echo "Synchronizing runc-$arch" - if [ $arch == "amd64" ]; then - curl -L -o binaries/compose/$COMPOSE_VERSION/$arch/docker-compose-linux-x86_64 \ - https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-linux-x86_64 - - qsctl cp binaries/compose/$COMPOSE_VERSION/$arch/docker-compose-linux-x86_64 \ - qs://kubernetes-release/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-linux-x86_64 \ - -c qsctl-config.yaml - - elif [ $arch == "arm64" ]; then - curl -L -o binaries/compose/$COMPOSE_VERSION/$arch/docker-compose-linux-aarch64 \ - https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-linux-aarch64 - - qsctl cp binaries/compose/$COMPOSE_VERSION/$arch/docker-compose-linux-aarch64 \ - qs://kubernetes-release/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-linux-aarch64 \ - -c qsctl-config.yaml - - fi - done - - rm -rf binaries -fi - -rm -rf qsctl-config.yaml - -# Sync NodeLocalDns Images -if [ $NODE_LOCAL_DNS_VERSION ]; then - skopeo sync --src docker --dest docker registry.k8s.io/dns/k8s-dns-node-cache:$NODE_LOCAL_DNS_VERSION docker.io/$DOCKERHUB_NAMESPACE/k8s-dns-node-cache:$NODE_LOCAL_DNS_VERSION --all - skopeo sync --src docker --dest docker registry.k8s.io/dns/k8s-dns-node-cache:$NODE_LOCAL_DNS_VERSION registry.cn-beijing.aliyuncs.com/$ALIYUNCS_NAMESPACE/k8s-dns-node-cache:$NODE_LOCAL_DNS_VERSION --all -fi - -# Sync Coredns Images -if [ $COREDNS_VERSION ]; then - skopeo sync --src docker --dest docker docker.io/coredns/coredns:$COREDNS_VERSION registry.cn-beijing.aliyuncs.com/$ALIYUNCS_NAMESPACE/coredns:$COREDNS_VERSION --all -fi - -# Sync Calico Images -if [ $CALICO_VERSION ]; then - skopeo sync --src docker --dest docker docker.io/calico/kube-controllers:$CALICO_VERSION registry.cn-beijing.aliyuncs.com/$ALIYUNCS_NAMESPACE/kube-controllers:$CALICO_VERSION --all - skopeo sync --src docker --dest docker docker.io/calico/cni:$CALICO_VERSION registry.cn-beijing.aliyuncs.com/$ALIYUNCS_NAMESPACE/cni:$CALICO_VERSION --all - skopeo sync --src docker --dest docker docker.io/calico/node:$CALICO_VERSION registry.cn-beijing.aliyuncs.com/$ALIYUNCS_NAMESPACE/node:$CALICO_VERSION --all - skopeo sync --src docker --dest docker docker.io/calico/pod2daemon-flexvol:$CALICO_VERSION registry.cn-beijing.aliyuncs.com/$ALIYUNCS_NAMESPACE/pod2daemon-flexvol:$CALICO_VERSION --all - skopeo sync --src docker --dest docker docker.io/calico/typha:$CALICO_VERSION registry.cn-beijing.aliyuncs.com/$ALIYUNCS_NAMESPACE/typha:$CALICO_VERSION --all -fi - -# Sync Kube-OVN Images -if [ $KUBE_OVN_VERSION ]; then - skopeo sync --src docker --dest docker docker.io/kubeovn/kube-ovn:$KUBE_OVN_VERSION registry.cn-beijing.aliyuncs.com/$ALIYUNCS_NAMESPACE/kube-ovn:$KUBE_OVN_VERSION --all - skopeo sync --src docker --dest docker docker.io/kubeovn/vpc-nat-gateway:$KUBE_OVN_VERSION registry.cn-beijing.aliyuncs.com/$ALIYUNCS_NAMESPACE/vpc-nat-gateway:$KUBE_OVN_VERSION --all -fi - -# Sync Cilium Images -if [ $CILIUM_VERSION ]; then - skopeo sync --src docker --dest docker docker.io/cilium/cilium:$CILIUM_VERSION registry.cn-beijing.aliyuncs.com/$ALIYUNCS_NAMESPACE/cilium:$CILIUM_VERSION --all - skopeo sync --src docker --dest docker docker.io/cilium/cilium-operator-generic:$CILIUM_VERSION registry.cn-beijing.aliyuncs.com/$ALIYUNCS_NAMESPACE/cilium-operator-generic:$CILIUM_VERSION --all -fi - -# Sync OpenEBS Images -if [ $OPENEBS_VERSION ]; then - skopeo sync --src docker --dest docker docker.io/openebs/provisioner-localpv:$OPENEBS_VERSION registry.cn-beijing.aliyuncs.com/$ALIYUNCS_NAMESPACE/provisioner-localpv:$OPENEBS_VERSION --all - skopeo sync --src docker --dest docker docker.io/openebs/linux-utils:$OPENEBS_VERSION registry.cn-beijing.aliyuncs.com/$ALIYUNCS_NAMESPACE/linux-utils:$OPENEBS_VERSION --all -fi - -# Sync Haproxy Images -if [ $HAPROXY_VERSION ]; then - skopeo sync --src docker --dest docker docker.io/library/haproxy:$HAPROXY_VERSION registry.cn-beijing.aliyuncs.com/$ALIYUNCS_NAMESPACE/haproxy:$HAPROXY_VERSION --all -fi - -# Sync Kube-vip Images -if [ $KUBEVIP_VERSION ]; then - skopeo sync --src docker --dest docker docker.io/plndr/kubevip:$KUBEVIP_VERSION registry.cn-beijing.aliyuncs.com/$ALIYUNCS_NAMESPACE/kubevip:$KUBEVIP_VERSION --all -fi diff --git a/hack/version.sh b/hack/version.sh deleted file mode 100755 index ea208410..00000000 --- a/hack/version.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2020 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -if [[ "${TRACE-0}" == "1" ]]; then - set -o xtrace -fi - -version::get_version_vars() { - # shellcheck disable=SC1083 - GIT_COMMIT="$(git rev-parse HEAD^{commit})" - - if git_status=$(git status --porcelain 2>/dev/null) && [[ -z ${git_status} ]]; then - GIT_TREE_STATE="clean" - else - GIT_TREE_STATE="dirty" - fi - - # stolen from k8s.io/hack/lib/version.sh - # Use git describe to find the version based on annotated tags. - if [[ -n ${GIT_VERSION-} ]] || GIT_VERSION=$(git describe --tags --abbrev=14 --match "v[0-9]*" "${GIT_COMMIT}" 2>/dev/null); then - # This translates the "git describe" to an actual semver.org - # compatible semantic version that looks something like this: - # v1.1.0-alpha.0.6+84c76d1142ea4d - # - # TODO: We continue calling this "git version" because so many - # downstream consumers are expecting it there. - # shellcheck disable=SC2001 - DASHES_IN_VERSION=$(echo "${GIT_VERSION}" | sed "s/[^-]//g") - if [[ "${DASHES_IN_VERSION}" == "---" ]] ; then - # We have distance to subversion (v1.1.0-subversion-1-gCommitHash) - # shellcheck disable=SC2001 - GIT_VERSION=$(echo "${GIT_VERSION}" | sed "s/-\([0-9]\{1,\}\)-g\([0-9a-f]\{14\}\)$/.\1\-\2/") - elif [[ "${DASHES_IN_VERSION}" == "--" ]] ; then - # We have distance to base tag (v1.1.0-1-gCommitHash) - # shellcheck disable=SC2001 - GIT_VERSION=$(echo "${GIT_VERSION}" | sed "s/-g\([0-9a-f]\{14\}\)$/-\1/") - fi - if [[ "${GIT_TREE_STATE}" == "dirty" ]]; then - # git describe --dirty only considers changes to existing files, but - # that is problematic since new untracked .go files affect the build, - # so use our idea of "dirty" from git status instead. - GIT_VERSION+="-dirty" - fi - - - # Try to match the "git describe" output to a regex to try to extract - # the "major" and "minor" versions and whether this is the exact tagged - # version or whether the tree is between two tagged versions. - if [[ "${GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?([-].*)?([+].*)?$ ]]; then - GIT_MAJOR=${BASH_REMATCH[1]} - GIT_MINOR=${BASH_REMATCH[2]} - fi - - # If GIT_VERSION is not a valid Semantic Version, then refuse to build. - if ! [[ "${GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then - echo "GIT_VERSION should be a valid Semantic Version. Current value: ${GIT_VERSION}" - echo "Please see more details here: https://semver.org" - exit 1 - fi - fi - - GIT_RELEASE_TAG=$(git describe --abbrev=0 --tags) - GIT_RELEASE_COMMIT=$(git rev-list -n 1 "${GIT_RELEASE_TAG}") -} - -# stolen from k8s.io/hack/lib/version.sh and modified -# Prints the value that needs to be passed to the -ldflags parameter of go build -version::ldflags() { - version::get_version_vars - - local -a ldflags - function add_ldflag() { - local key=${1} - local val=${2} - ldflags+=( - "-X 'github.com/kubesphere/kubekey/v3/version.${key}=${val}'" - ) - } - - add_ldflag "buildDate" "$(date ${SOURCE_DATE_EPOCH:+"--date=@${SOURCE_DATE_EPOCH}"} -u +'%Y-%m-%dT%H:%M:%SZ')" - add_ldflag "gitCommit" "${GIT_COMMIT}" - add_ldflag "gitTreeState" "${GIT_TREE_STATE}" - add_ldflag "gitMajor" "${GIT_MAJOR}" - add_ldflag "gitMinor" "${GIT_MINOR}" - add_ldflag "gitVersion" "${GIT_VERSION}" - add_ldflag "gitReleaseCommit" "${GIT_RELEASE_COMMIT}" - - # The -ldflags parameter takes a single string, so join the output. - echo "${ldflags[*]-}" -} - -version::ldflags diff --git a/main.go b/main.go deleted file mode 100644 index 5dbe8978..00000000 --- a/main.go +++ /dev/null @@ -1,263 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package main -package main - -import ( - "flag" - "fmt" - "math/rand" - "os" - "time" - - "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth" - "k8s.io/client-go/tools/leaderelection/resourcelock" - "k8s.io/klog/v2" - "k8s.io/klog/v2/klogr" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/remote" - controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" - "sigs.k8s.io/cluster-api/util/record" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/controller" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" - "github.com/kubesphere/kubekey/v3/controllers" - //+kubebuilder:scaffold:imports -) - -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") -) - -func init() { - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(clusterv1.AddToScheme(scheme)) - utilruntime.Must(infrav1.AddToScheme(scheme)) - utilruntime.Must(controlplanev1.AddToScheme(scheme)) - //+kubebuilder:scaffold:scheme -} - -var ( - metricsAddr string - enableLeaderElection bool - leaderElectionNamespace string - healthAddr string - watchFilterValue string - kkClusterConcurrency int - kkInstanceConcurrency int - kkMachineConcurrency int - syncPeriod time.Duration - watchNamespace string - dataDir string -) - -func main() { - klog.InitFlags(nil) - - rand.Seed(time.Now().UnixNano()) - initFlags(pflag.CommandLine) - pflag.CommandLine.AddGoFlagSet(flag.CommandLine) - pflag.Parse() - - ctrl.SetLogger(klogr.New()) - - ctx := ctrl.SetupSignalHandler() - - restConfig := ctrl.GetConfigOrDie() - restConfig.UserAgent = "cluster-api-provider-kk-controller" - mgr, err := ctrl.NewManager(restConfig, ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "controller-leader-election-capkk", - LeaderElectionResourceLock: resourcelock.LeasesResourceLock, - LeaderElectionNamespace: leaderElectionNamespace, - SyncPeriod: &syncPeriod, - Namespace: watchNamespace, - Port: 9443, - HealthProbeBindAddress: healthAddr, - }) - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) - } - - log := ctrl.Log.WithName("remote").WithName("ClusterCacheTracker") - tracker, err := remote.NewClusterCacheTracker( - mgr, - remote.ClusterCacheTrackerOptions{ - Log: &log, - Indexes: remote.DefaultIndexes, - }, - ) - if err != nil { - setupLog.Error(err, "unable to create cluster cache tracker") - os.Exit(1) - } - - // Initialize event recorder. - record.InitFromRecorder(mgr.GetEventRecorderFor("kk-controller")) - - if err = (&controllers.KKClusterReconciler{ - Client: mgr.GetClient(), - Recorder: mgr.GetEventRecorderFor("kkcluster-controller"), - Scheme: mgr.GetScheme(), - WatchFilterValue: watchFilterValue, - DataDir: dataDir, - }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: kkClusterConcurrency, RecoverPanic: true}); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "KKCluster") - os.Exit(1) - } - if err = (&controllers.KKMachineReconciler{ - Client: mgr.GetClient(), - Recorder: mgr.GetEventRecorderFor("kkmachine-controller"), - Scheme: mgr.GetScheme(), - Tracker: tracker, - WatchFilterValue: watchFilterValue, - DataDir: dataDir, - }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: kkMachineConcurrency, RecoverPanic: true}); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "KKMachine") - os.Exit(1) - } - if err = (&controllers.KKInstanceReconciler{ - Client: mgr.GetClient(), - Recorder: mgr.GetEventRecorderFor("kkinstance-controller"), - Scheme: mgr.GetScheme(), - Tracker: tracker, - WatchFilterValue: watchFilterValue, - DataDir: dataDir, - }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: kkInstanceConcurrency, RecoverPanic: true}); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "KKInstance") - os.Exit(1) - } - - if err = (&infrav1.KKCluster{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "KKCluster") - os.Exit(1) - } - if err = (&infrav1.KKClusterTemplate{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "KKClusterTemplate") - os.Exit(1) - } - if err = (&infrav1.KKMachine{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "KKMachine") - os.Exit(1) - } - if err = (&infrav1.KKMachineTemplate{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "KKMachineTemplate") - os.Exit(1) - } - if err = (&infrav1.KKInstance{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "KKInstance") - os.Exit(1) - } - //+kubebuilder:scaffold:builder - - if err := mgr.AddHealthzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { - setupLog.Error(err, "unable to create health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { - setupLog.Error(err, "unable to create ready check") - os.Exit(1) - } - - setupLog.Info("starting manager") - if err := mgr.Start(ctx); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) - } -} - -func initFlags(fs *pflag.FlagSet) { - fs.StringVar( - &metricsAddr, - "metrics-bind-address", - ":8080", - "The address the metric endpoint binds to.", - ) - - fs.BoolVar( - &enableLeaderElection, - "leader-elect", - false, - "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.", - ) - - fs.StringVar( - &watchNamespace, - "namespace", - "", - "Namespace that the controller watches to reconcile cluster-api objects. If unspecified, the controller watches for cluster-api objects across all namespaces.", - ) - - fs.StringVar( - &leaderElectionNamespace, - "leader-elect-namespace", - "", - "Namespace that the controller performs leader election in. If unspecified, the controller will discover which namespace it is running in.", - ) - - fs.DurationVar(&syncPeriod, - "sync-period", - 10*time.Minute, - "The minimum interval at which watched resources are reconciled.", - ) - - fs.IntVar(&kkClusterConcurrency, - "kkcluster-concurrency", - 5, - "Number of KKClusters to process simultaneously.", - ) - - fs.IntVar(&kkInstanceConcurrency, - "kkinstance-concurrency", - 10, - "Number of KKInstance to process simultaneously.", - ) - - fs.IntVar(&kkMachineConcurrency, - "kkmachine-concurrency", - 10, - "Number of KKMachines to process simultaneously.", - ) - - fs.StringVar(&healthAddr, - "health-addr", - ":9440", - "The address the health endpoint binds to.") - - fs.StringVar( - &watchFilterValue, - "watch-filter", - "", - fmt.Sprintf("Label value that the controller watches to reconcile cluster-api objects. Label key is always %s. If unspecified, the controller watches for all cluster-api objects.", clusterv1.WatchLabel), - ) - - fs.StringVar( - &dataDir, - "data-dir", - "", - "The KubeKey data dir.", - ) -} diff --git a/metadata.yaml b/metadata.yaml deleted file mode 100644 index e0ec3ba5..00000000 --- a/metadata.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# maps release series of major.minor to cluster-api contract version -# the contract version may change between minor or major versions, but *not* -# between patch versions. -# -# update this file only when a new major or minor version is released -apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 -releaseSeries: - - major: 3 - minor: 0 - contract: v1beta1 diff --git a/pkg/clients/ssh/client.go b/pkg/clients/ssh/client.go deleted file mode 100644 index 323033c6..00000000 --- a/pkg/clients/ssh/client.go +++ /dev/null @@ -1,576 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package ssh - -import ( - "bufio" - "fmt" - "io" - "net" - "os" - "path" - "path/filepath" - "strconv" - "strings" - "sync" - "time" - - "github.com/go-logr/logr" - "github.com/pkg/errors" - "github.com/pkg/sftp" - "golang.org/x/crypto/ssh" - "k8s.io/klog/v2/klogr" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" - "github.com/kubesphere/kubekey/v3/pkg/util/filesystem" -) - -// Default values. -const ( - DefaultSSHPort = 22 - DefaultTimeout = 15 - ROOT = "root" -) - -// Client is a wrapper around the SSH client that provides a few helper. -type Client struct { - logr.Logger - mu sync.Mutex - user string - password string - port *int - privateKey string - privateKeyPath string - timeout *time.Duration - host string - sshClient *ssh.Client - sftpClient *sftp.Client - fs filesystem.Interface -} - -// NewClient returns a new client given ssh information. -func NewClient(host string, auth infrav1.Auth, log *logr.Logger) Interface { - if log == nil { - l := klogr.New() - log = &l - } - if auth.User == "" { - auth.User = ROOT - } - - port := DefaultSSHPort - if auth.Port == nil { - auth.Port = &port - } - - timeout := time.Duration(DefaultTimeout) * time.Second - if auth.Timeout == nil { - auth.Timeout = &timeout - } - - return &Client{ - user: auth.User, - password: auth.Password, - port: auth.Port, - privateKey: auth.PrivateKey, - privateKeyPath: auth.PrivateKeyPath, - timeout: auth.Timeout, - host: host, - fs: filesystem.NewFileSystem(), - Logger: *log, - } -} - -// Connect connects to the host using the provided ssh information. -func (c *Client) Connect() error { - authMethods, err := c.authMethod(c.password, c.privateKey, c.privateKeyPath) - if err != nil { - return errors.Wrap(err, "The given SSH key could not be parsed") - } - - sshConfig := &ssh.ClientConfig{ - User: c.user, - Timeout: *c.timeout, - Auth: authMethods, - HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { - return nil - }, - } - - endpoint := net.JoinHostPort(c.host, strconv.Itoa(*c.port)) - sshClient, err := ssh.Dial("tcp", endpoint, sshConfig) - if err != nil { - return errors.Wrapf(err, "could not establish connection to %s", endpoint) - } - - c.sshClient = sshClient - return nil -} - -// ConnectSftpClient connects to the host sftp client using the provided ssh information. -func (c *Client) ConnectSftpClient(opts ...sftp.ClientOption) error { - var ( - sftpClient *sftp.Client - err error - ) - - sftpClient, err = sftp.NewClient(c.sshClient, opts...) - c.sftpClient = sftpClient - return err -} - -// Close closes the underlying ssh and sftp connection. -func (c *Client) Close() { - c.mu.Lock() - defer c.mu.Unlock() - - if c.sshClient == nil && c.sftpClient == nil { - return - } - - if c.sshClient != nil { - _ = c.sshClient.Close() - c.sshClient = nil - } - if c.sftpClient != nil { - _ = c.sftpClient.Close() - c.sftpClient = nil - } -} - -func (c *Client) authMethod(password, privateKey, privateKeyPath string) (auths []ssh.AuthMethod, err error) { - if privateKey != "" || privateKeyPath != "" { - am, err := c.privateKeyMethod(privateKey, privateKeyPath) - if err != nil { - return auths, err - } - auths = append(auths, am) - } - if password != "" { - auths = append(auths, ssh.Password(password)) - } - return auths, nil -} - -func (c *Client) privateKeyMethod(privateKey, privateKeyPath string) (am ssh.AuthMethod, err error) { - var signer ssh.Signer - - if fileExist(privateKeyPath) { - content, err := os.ReadFile(filepath.Clean(privateKeyPath)) - if err != nil { - return nil, err - } - if privateKey == "" { - signer, err = ssh.ParsePrivateKey(content) - if err != nil { - return nil, err - } - } else { - passphrase := []byte(privateKey) - signer, err = ssh.ParsePrivateKeyWithPassphrase(content, passphrase) - if err != nil { - return nil, err - } - } - } else { - signer, err = ssh.ParsePrivateKey([]byte(privateKey)) - if err != nil { - return nil, err - } - } - - return ssh.PublicKeys(signer), nil -} - -func (c *Client) session() (*ssh.Session, error) { - c.mu.Lock() - defer c.mu.Unlock() - - if c.sshClient == nil { - return nil, errors.New("connection closed") - } - - sess, err := c.sshClient.NewSession() - if err != nil { - return nil, err - } - - modes := ssh.TerminalModes{ - ssh.ECHO: 0, // disable echoing - ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud - ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud - } - - err = sess.RequestPty("xterm", 100, 50, modes) - if err != nil { - return nil, err - } - - return sess, nil -} - -// Cmd executes a command on the remote host. -func (c *Client) Cmd(cmd string) (string, error) { - if err := c.Connect(); err != nil { - return "", errors.Wrapf(err, "[%s] connect ssh client failed", c.host) - } - session, err := c.session() - if err != nil { - return "", errors.Wrapf(err, "[%s] create ssh session failed", c.host) - } - defer session.Close() - defer c.sshClient.Close() - - c.Logger.V(4).Info(fmt.Sprintf("cmd: %s", cmd)) - - output, err := session.CombinedOutput(cmd) - if err != nil { - return "", errors.Wrapf(err, "[%s] run command failed", c.host) - } - - return string(output), nil -} - -// Cmdf execute a formatting command according to the format specifier. -func (c *Client) Cmdf(cmd string, a ...any) (string, error) { - return c.Cmd(fmt.Sprintf(cmd, a...)) -} - -// SudoCmd executes a command on the remote host with sudo. -func (c *Client) SudoCmd(cmd string) (string, error) { - if err := c.Connect(); err != nil { - return "", errors.Wrapf(err, "[%s] connect ssh client failed", c.host) - } - defer c.sshClient.Close() - return c.sudoCmd(cmd) -} - -func (c *Client) sudoCmd(cmd string) (string, error) { - session, err := c.session() - if err != nil { - return "", errors.Wrapf(err, "[%s] create ssh session failed", c.host) - } - defer session.Close() - - cmd = SudoPrefix(cmd) - c.Logger.V(4).Info(fmt.Sprintf("cmd: %s", cmd)) - - in, err := session.StdinPipe() - if err != nil { - return "", err - } - - out, err := session.StdoutPipe() - if err != nil { - return "", err - } - - if err := session.Start(cmd); err != nil { - return "", err - } - var ( - output []byte - line = "" - r = bufio.NewReader(out) - ) - - for { - b, err := r.ReadByte() - if err != nil { - break - } - - output = append(output, b) - - if b == byte('\n') { - line = "" - continue - } - - line += string(b) - - if (strings.HasPrefix(line, "[sudo] password for ") || strings.HasPrefix(line, "Password")) && strings.HasSuffix(line, ": ") { - _, err = in.Write([]byte(c.password + "\n")) - if err != nil { - break - } - } - } - - outStr := strings.TrimPrefix(string(output), fmt.Sprintf("[sudo] password for %s:", c.user)) - err = session.Wait() - if err != nil { - return strings.TrimSpace(outStr), errors.Wrap(err, strings.TrimSpace(outStr)) - } - return strings.TrimSpace(outStr), nil -} - -// SudoCmdf executes a formatting command on the remote host with sudo. -func (c *Client) SudoCmdf(cmd string, a ...any) (string, error) { - return c.SudoCmd(fmt.Sprintf(cmd, a...)) -} - -// Copy copies a file to the remote host. -func (c *Client) Copy(src, dst string) error { - if c.user == ROOT { - return c.copy(src, dst) - } - return c.sudoCopy(src, dst) -} - -func (c *Client) sudoCopy(src, dst string) error { - // scp to tmp dir - remoteTmp := filepath.Join("/tmp/kubekey", dst) - if err := c.copy(src, remoteTmp); err != nil { - return err - } - - baseRemotePath := filepath.Dir(dst) - if err := c.mkdirAll(baseRemotePath, ""); err != nil { - return err - } - if _, err := c.SudoCmdf("mv -f %s %s", remoteTmp, dst); err != nil { - return errors.Wrapf(err, "[%s] mv -f %s %s failed", c.host, remoteTmp, dst) - } - if _, err := c.SudoCmd("rm -rf /tmp/kubekey*"); err != nil { - return errors.Wrapf(err, "[%s] rm -rf /tmp/kubekey* failed", c.host) - } - return nil -} - -func (c *Client) copy(src, dst string) error { - baseRemoteFilePath := filepath.Dir(dst) - _ = c.mkdirAll(baseRemoteFilePath, "777") - - if err := c.Connect(); err != nil { - return errors.Wrapf(err, "[%s] connect ssh client failed", c.host) - } - if err := c.ConnectSftpClient(); err != nil { - return errors.Wrapf(err, "[%s] connect sftp client failed", c.host) - } - defer c.sshClient.Close() - defer c.sftpClient.Close() - - if err := c.copyLocalFileToRemote(src, dst); err != nil { - return errors.Wrapf(err, "[%s] copy file failed", c.host) - } - return nil -} - -func (c *Client) copyLocalFileToRemote(src, dst string) error { - // check remote file md5 first - var ( - srcMd5, dstMd5 string - ) - cleanSrc := filepath.Clean(src) - srcMd5 = c.fs.MD5Sum(cleanSrc) - if exist, err := c.remoteFileExist(dst); err != nil { - return err - } else if exist { - dstMd5 = c.remoteMd5Sum(dst) - if srcMd5 == dstMd5 { - c.Logger.V(4).Info(fmt.Sprintf("remote file %s md5 value is the same as local file, skip scp", dst)) - return nil - } - } - - srcFile, err := os.Open(cleanSrc) - if err != nil { - return errors.Wrapf(err, "open local file %s failed", cleanSrc) - } - defer srcFile.Close() - - fileStat, err := srcFile.Stat() - if err != nil { - return fmt.Errorf("get file stat failed %v", err) - } - if fileStat.IsDir() { - return fmt.Errorf("the source %s is not a file", cleanSrc) - } - - // the dst file mod will be 0666 - dstFile, err := c.sftpClient.Create(dst) - if err != nil { - return errors.Wrapf(err, "[%s] create remote file %s failed", c.host, dst) - } - if err := dstFile.Chmod(fileStat.Mode()); err != nil { - return fmt.Errorf("chmod remote file failed %v", err) - } - defer dstFile.Close() - - _, err = io.Copy(dstFile, srcFile) - if err != nil { - return errors.Wrapf(err, "[%s] io copy file %s to remote %s failed", c.host, cleanSrc, dst) - } - - dstMd5 = c.remoteMd5Sum(dst) - if srcMd5 != dstMd5 { - return fmt.Errorf("validate md5sum failed %s != %s", srcMd5, dstMd5) - } - return nil -} - -// Fetch fetches a file from the remote host. -func (c *Client) Fetch(local, remote string) error { - if c.user == ROOT { - return c.fetch(local, remote) - } - return c.sudoFetch(local, remote) -} - -func (c *Client) sudoFetch(local, remote string) error { - remoteTmp := filepath.Join("/tmp/kubekey", filepath.Base(remote)) - baseRemotePath := filepath.Dir(remoteTmp) - if err := c.mkdirAll(baseRemotePath, "777"); err != nil { - return err - } - if _, err := c.SudoCmdf("cp %s %s", remote, remoteTmp); err != nil { - return errors.Wrapf(err, "[%s] cp %s %s failed", c.host, remote, remoteTmp) - } - - if err := c.fetch(local, remoteTmp); err != nil { - return err - } - if _, err := c.SudoCmd("rm -rf /tmp/kubekey*"); err != nil { - return errors.Wrapf(err, "[%s] rm -rf /tmp/kubekey* failed", c.host) - } - return nil -} - -func (c *Client) fetch(local, remote string) error { - if err := c.Connect(); err != nil { - return errors.Wrapf(err, "[%s] connect ssh client failed", c.host) - } - - if err := c.ConnectSftpClient(); err != nil { - return errors.Wrapf(err, "[%s] connect sftp client failed", c.host) - } - defer c.sshClient.Close() - defer c.sftpClient.Close() - - ok, err := c.RemoteFileExist(remote) - if err != nil { - return errors.Wrapf(err, "[%s] check remote file failed", c.host) - } - if !ok { - return errors.Errorf("[%s] remote file %s not exist", c.host, remote) - } - - // open remote source file - srcFile, err := c.sftpClient.Open(remote) - if err != nil { - return fmt.Errorf("open remote file failed %v, remote path: %s", err, remote) - } - defer func() { - if err := srcFile.Close(); err != nil { - c.Logger.Error(err, "failed to close file") - } - }() - - err = os.MkdirAll(filepath.Dir(local), os.ModePerm) - if err != nil { - return err - } - - dstFile, err := os.Create(filepath.Clean(local)) - if err != nil { - return fmt.Errorf("create local file failed %v", err) - } - defer func() { - if err := dstFile.Close(); err != nil { - c.Logger.Error(err, "failed to close file") - } - }() - - _, err = srcFile.WriteTo(dstFile) - return err -} - -func (c *Client) remoteMd5Sum(dst string) string { - cmd := fmt.Sprintf("md5sum %s | cut -d\" \" -f1", dst) - remoteMd5, err := c.sudoCmd(cmd) - if err != nil { - c.Logger.Error(err, fmt.Sprintf("sum remote file md5 failed, output: %s", remoteMd5)) - return "" - } - return remoteMd5 -} - -// RemoteFileExist checks if a file exists on the remote host. -func (c *Client) RemoteFileExist(remote string) (bool, error) { - if err := c.Connect(); err != nil { - return false, errors.Wrapf(err, "[%s] connect failed", c.host) - } - defer c.sshClient.Close() - return c.remoteFileExist(remote) -} - -func (c *Client) remoteFileExist(remote string) (bool, error) { - remoteFileName := path.Base(remote) - remoteFileDirName := path.Dir(remote) - - remoteFileCommand := fmt.Sprintf("ls -l %s/%s 2>/dev/null |wc -l", remoteFileDirName, remoteFileName) - - out, err := c.sudoCmd(remoteFileCommand) - if err != nil { - return false, err - } - count, err := strconv.Atoi(strings.TrimSpace(out)) - if err != nil { - return false, err - } - return count != 0, nil -} - -func (c *Client) mkdirAll(path, mode string) error { - if mode == "" { - mode = "775" - } - if _, err := c.SudoCmdf("mkdir -p -m %s %s", mode, path); err != nil { - return errors.Wrapf(err, "[%s] mkdir -p -m %s %s failed", c.host, mode, path) - } - return nil -} - -// Ping checks if the remote host is reachable. -func (c *Client) Ping() error { - if err := c.Connect(); err != nil { - return errors.Wrapf(err, "[%s] connect failed", c.host) - } - defer c.Close() - return nil -} - -// Host returns the host name of the ssh client. -func (c *Client) Host() string { - return c.host -} - -// Fs returns the filesystem of the ssh client. -func (c *Client) Fs() filesystem.Interface { - return c.fs -} - -func fileExist(path string) bool { - _, err := os.Stat(path) - return err == nil || os.IsExist(err) -} - -// SudoPrefix returns the prefix for sudo commands. -func SudoPrefix(cmd string) string { - return fmt.Sprintf("sudo -E /bin/bash <&2 -} -fatal() -{ - echo '[ERROR] ' "$@" >&2 - exit 1 -} - -# --- fatal if no systemd or openrc --- -verify_system() { - if [ -x /sbin/openrc-run ]; then - HAS_OPENRC=true - return - fi - if [ -x /bin/systemctl ] || type systemctl > /dev/null 2>&1; then - HAS_SYSTEMD=true - return - fi - fatal 'Can not find systemd or openrc to use as a process supervisor for k3s' -} - -# --- add quotes to command arguments --- -quote() { - for arg in "$@"; do - printf '%s\n' "$arg" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" - done -} - -# --- add indentation and trailing slash to quoted args --- -quote_indent() { - printf ' \\\n' - for arg in "$@"; do - printf '\t%s \\\n' "$(quote "$arg")" - done -} - -# --- escape most punctuation characters, except quotes, forward slash, and space --- -escape() { - printf '%s' "$@" | sed -e 's/\([][!#$%&()*;<=>?\_`{|}]\)/\\\1/g;' -} - -# --- escape double quotes --- -escape_dq() { - printf '%s' "$@" | sed -e 's/"/\\"/g' -} - -# --- ensures $K3S_URL is empty or begins with https://, exiting fatally otherwise --- -verify_k3s_url() { - case "${K3S_URL}" in - "") - ;; - https://*) - ;; - *) - fatal "Only https:// URLs are supported for K3S_URL (have ${K3S_URL})" - ;; - esac -} - -# --- define needed environment variables --- -setup_env() { - # --- use command args if passed or create default --- - case "$1" in - # --- if we only have flags discover if command should be server or agent --- - (-*|"") - if [ -z "${K3S_URL}" ]; then - CMD_K3S=server - else - if [ -z "${K3S_TOKEN}" ] && [ -z "${K3S_TOKEN_FILE}" ] && [ -z "${K3S_CLUSTER_SECRET}" ]; then - fatal "Defaulted k3s exec command to 'agent' because K3S_URL is defined, but K3S_TOKEN, K3S_TOKEN_FILE or K3S_CLUSTER_SECRET is not defined." - fi - CMD_K3S=agent - fi - ;; - # --- command is provided --- - (*) - CMD_K3S=$1 - shift - ;; - esac - - verify_k3s_url - - CMD_K3S_EXEC="${CMD_K3S}$(quote_indent "$@")" - - # --- use systemd name if defined or create default --- - if [ -n "${INSTALL_K3S_NAME}" ]; then - SYSTEM_NAME=k3s-${INSTALL_K3S_NAME} - else - if [ "${CMD_K3S}" = server ]; then - SYSTEM_NAME=k3s - else - SYSTEM_NAME=k3s-${CMD_K3S} - fi - fi - - # --- check for invalid characters in system name --- - valid_chars=$(printf '%s' "${SYSTEM_NAME}" | sed -e 's/[][!#$%&()*;<=>?\_`{|}/[:space:]]/^/g;' ) - if [ "${SYSTEM_NAME}" != "${valid_chars}" ]; then - invalid_chars=$(printf '%s' "${valid_chars}" | sed -e 's/[^^]/ /g') - fatal "Invalid characters for system name: - ${SYSTEM_NAME} - ${invalid_chars}" - fi - - # --- use sudo if we are not already root --- - SUDO=sudo - if [ $(id -u) -eq 0 ]; then - SUDO= - fi - - # --- use systemd type if defined or create default --- - if [ -n "${INSTALL_K3S_TYPE}" ]; then - SYSTEMD_TYPE=${INSTALL_K3S_TYPE} - else - SYSTEMD_TYPE=notify - fi - - # --- use binary install directory if defined or create default --- - if [ -n "${INSTALL_K3S_BIN_DIR}" ]; then - BIN_DIR=${INSTALL_K3S_BIN_DIR} - else - # --- use /usr/local/bin if root can write to it, otherwise use /opt/bin if it exists - BIN_DIR=/usr/local/bin - if ! $SUDO sh -c "touch ${BIN_DIR}/k3s-ro-test && rm -rf ${BIN_DIR}/k3s-ro-test"; then - if [ -d /opt/bin ]; then - BIN_DIR=/opt/bin - fi - fi - fi - - # --- use systemd directory if defined or create default --- - if [ -n "${INSTALL_K3S_SYSTEMD_DIR}" ]; then - SYSTEMD_DIR="${INSTALL_K3S_SYSTEMD_DIR}" - else - SYSTEMD_DIR=/etc/systemd/system - fi - - # --- set related files from system name --- - SERVICE_K3S=${SYSTEM_NAME}.service - UNINSTALL_K3S_SH=${UNINSTALL_K3S_SH:-${BIN_DIR}/${SYSTEM_NAME}-uninstall.sh} - KILLALL_K3S_SH=${KILLALL_K3S_SH:-${BIN_DIR}/k3s-killall.sh} - - # --- use service or environment location depending on systemd/openrc --- - if [ "${HAS_SYSTEMD}" = true ]; then - FILE_K3S_SERVICE=${SYSTEMD_DIR}/${SERVICE_K3S} - FILE_K3S_ENV=${SYSTEMD_DIR}/${SERVICE_K3S}.env - elif [ "${HAS_OPENRC}" = true ]; then - $SUDO mkdir -p /etc/rancher/k3s - FILE_K3S_SERVICE=/etc/init.d/${SYSTEM_NAME} - FILE_K3S_ENV=/etc/rancher/k3s/${SYSTEM_NAME}.env - fi - - # --- get hash of config & exec for currently installed k3s --- - PRE_INSTALL_HASHES=$(get_installed_hashes) - - # --- if bin directory is read only skip download --- - if [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ]; then - INSTALL_K3S_SKIP_DOWNLOAD=true - fi - - # --- setup channel values - INSTALL_K3S_CHANNEL_URL=${INSTALL_K3S_CHANNEL_URL:-'https://update.k3s.io/v1-release/channels'} - INSTALL_K3S_CHANNEL=${INSTALL_K3S_CHANNEL:-'stable'} -} - -# --- check if skip download environment variable set --- -can_skip_download_binary() { - if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != true ] && [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != binary ]; then - return 1 - fi -} - -can_skip_download_selinux() { - if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != true ] && [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != selinux ]; then - return 1 - fi -} - -# --- verify an executable k3s binary is installed --- -verify_k3s_is_executable() { - if [ ! -x ${BIN_DIR}/k3s ]; then - fatal "Executable k3s binary not found at ${BIN_DIR}/k3s" - fi -} - -# --- set arch and suffix, fatal if architecture not supported --- -setup_verify_arch() { - if [ -z "$ARCH" ]; then - ARCH=$(uname -m) - fi - case $ARCH in - amd64) - ARCH=amd64 - SUFFIX= - ;; - x86_64) - ARCH=amd64 - SUFFIX= - ;; - arm64) - ARCH=arm64 - SUFFIX=-${ARCH} - ;; - s390x) - ARCH=s390x - SUFFIX=-${ARCH} - ;; - aarch64) - ARCH=arm64 - SUFFIX=-${ARCH} - ;; - arm*) - ARCH=arm - SUFFIX=-${ARCH}hf - ;; - *) - fatal "Unsupported architecture $ARCH" - esac -} - -# --- verify existence of network downloader executable --- -verify_downloader() { - # Return failure if it doesn't exist or is no executable - [ -x "$(command -v $1)" ] || return 1 - - # Set verified executable as our downloader program and return success - DOWNLOADER=$1 - return 0 -} - -# --- create temporary directory and cleanup when done --- -setup_tmp() { - TMP_DIR=$(mktemp -d -t k3s-install.XXXXXXXXXX) - TMP_HASH=${TMP_DIR}/k3s.hash - TMP_BIN=${TMP_DIR}/k3s.bin - cleanup() { - code=$? - set +e - trap - EXIT - rm -rf ${TMP_DIR} - exit $code - } - trap cleanup INT EXIT -} - -# --- use desired k3s version if defined or find version from channel --- -get_release_version() { - if [ -n "${INSTALL_K3S_COMMIT}" ]; then - VERSION_K3S="commit ${INSTALL_K3S_COMMIT}" - elif [ -n "${INSTALL_K3S_VERSION}" ]; then - VERSION_K3S=${INSTALL_K3S_VERSION} - else - info "Finding release for channel ${INSTALL_K3S_CHANNEL}" - version_url="${INSTALL_K3S_CHANNEL_URL}/${INSTALL_K3S_CHANNEL}" - case $DOWNLOADER in - curl) - VERSION_K3S=$(curl -w '%{url_effective}' -L -s -S ${version_url} -o /dev/null | sed -e 's|.*/||') - ;; - wget) - VERSION_K3S=$(wget -SqO /dev/null ${version_url} 2>&1 | grep -i Location | sed -e 's|.*/||') - ;; - *) - fatal "Incorrect downloader executable '$DOWNLOADER'" - ;; - esac - fi - info "Using ${VERSION_K3S} as release" -} - -# --- download from github url --- -download() { - [ $# -eq 2 ] || fatal 'download needs exactly 2 arguments' - - case $DOWNLOADER in - curl) - curl -o $1 -sfL $2 - ;; - wget) - wget -qO $1 $2 - ;; - *) - fatal "Incorrect executable '$DOWNLOADER'" - ;; - esac - - # Abort if download command failed - [ $? -eq 0 ] || fatal 'Download failed' -} - -# --- download hash from github url --- -download_hash() { - if [ -n "${INSTALL_K3S_COMMIT}" ]; then - HASH_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT}.sha256sum - else - HASH_URL=${GITHUB_URL}/download/${VERSION_K3S}/sha256sum-${ARCH}.txt - fi - info "Downloading hash ${HASH_URL}" - download ${TMP_HASH} ${HASH_URL} - HASH_EXPECTED=$(grep " k3s${SUFFIX}$" ${TMP_HASH}) - HASH_EXPECTED=${HASH_EXPECTED%%[[:blank:]]*} -} - -# --- check hash against installed version --- -installed_hash_matches() { - if [ -x ${BIN_DIR}/k3s ]; then - HASH_INSTALLED=$(sha256sum ${BIN_DIR}/k3s) - HASH_INSTALLED=${HASH_INSTALLED%%[[:blank:]]*} - if [ "${HASH_EXPECTED}" = "${HASH_INSTALLED}" ]; then - return - fi - fi - return 1 -} - -# --- download binary from github url --- -download_binary() { - if [ -n "${INSTALL_K3S_COMMIT}" ]; then - BIN_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT} - else - BIN_URL=${GITHUB_URL}/download/${VERSION_K3S}/k3s${SUFFIX} - fi - info "Downloading binary ${BIN_URL}" - download ${TMP_BIN} ${BIN_URL} -} - -# --- verify downloaded binary hash --- -verify_binary() { - info "Verifying binary download" - HASH_BIN=$(sha256sum ${TMP_BIN}) - HASH_BIN=${HASH_BIN%%[[:blank:]]*} - if [ "${HASH_EXPECTED}" != "${HASH_BIN}" ]; then - fatal "Download sha256 does not match ${HASH_EXPECTED}, got ${HASH_BIN}" - fi -} - -# --- setup permissions and move binary to system directory --- -setup_binary() { - chmod 755 ${TMP_BIN} - info "Installing k3s to ${BIN_DIR}/k3s" - $SUDO chown root:root ${TMP_BIN} - $SUDO mv -f ${TMP_BIN} ${BIN_DIR}/k3s -} - -# --- setup selinux policy --- -setup_selinux() { - case ${INSTALL_K3S_CHANNEL} in - *testing) - rpm_channel=testing - ;; - *latest) - rpm_channel=latest - ;; - *) - rpm_channel=stable - ;; - esac - - rpm_site="rpm.rancher.io" - if [ "${rpm_channel}" = "testing" ]; then - rpm_site="rpm-testing.rancher.io" - fi - - [ -r /etc/os-release ] && . /etc/os-release - if [ "${ID_LIKE%%[ ]*}" = "suse" ]; then - rpm_target=sle - rpm_site_infix=microos - package_installer=zypper - elif [ "${VERSION_ID%%.*}" = "7" ]; then - rpm_target=el7 - rpm_site_infix=centos/7 - package_installer=yum - else - rpm_target=el8 - rpm_site_infix=centos/8 - package_installer=yum - fi - - if [ "${package_installer}" = "yum" ] && [ -x /usr/bin/dnf ]; then - package_installer=dnf - fi - - policy_hint="please install: - ${package_installer} install -y container-selinux - ${package_installer} install -y https://${rpm_site}/k3s/${rpm_channel}/common/${rpm_site_infix}/noarch/k3s-selinux-0.4-1.${rpm_target}.noarch.rpm -" - - if [ "$INSTALL_K3S_SKIP_SELINUX_RPM" = true ] || can_skip_download_selinux || [ ! -d /usr/share/selinux ]; then - info "Skipping installation of SELinux RPM" - elif [ "${ID_LIKE:-}" != coreos ] && [ "${VARIANT_ID:-}" != coreos ]; then - install_selinux_rpm ${rpm_site} ${rpm_channel} ${rpm_target} ${rpm_site_infix} - fi - - policy_error=fatal - if [ "$INSTALL_K3S_SELINUX_WARN" = true ] || [ "${ID_LIKE:-}" = coreos ] || [ "${VARIANT_ID:-}" = coreos ]; then - policy_error=warn - fi - - if ! $SUDO chcon -u system_u -r object_r -t container_runtime_exec_t ${BIN_DIR}/k3s >/dev/null 2>&1; then - if $SUDO grep '^\s*SELINUX=enforcing' /etc/selinux/config >/dev/null 2>&1; then - $policy_error "Failed to apply container_runtime_exec_t to ${BIN_DIR}/k3s, ${policy_hint}" - fi - elif [ ! -f /usr/share/selinux/packages/k3s.pp ]; then - if [ -x /usr/sbin/transactional-update ]; then - warn "Please reboot your machine to activate the changes and avoid data loss." - else - $policy_error "Failed to find the k3s-selinux policy, ${policy_hint}" - fi - fi -} - -install_selinux_rpm() { - if [ -r /etc/redhat-release ] || [ -r /etc/centos-release ] || [ -r /etc/oracle-release ] || [ "${ID_LIKE%%[ ]*}" = "suse" ]; then - repodir=/etc/yum.repos.d - if [ -d /etc/zypp/repos.d ]; then - repodir=/etc/zypp/repos.d - fi - set +o noglob - $SUDO rm -f ${repodir}/rancher-k3s-common*.repo - set -o noglob - if [ -r /etc/redhat-release ] && [ "${3}" = "el7" ]; then - $SUDO yum install -y yum-utils - $SUDO yum-config-manager --enable rhel-7-server-extras-rpms - fi - $SUDO tee ${repodir}/rancher-k3s-common.repo >/dev/null << EOF -[rancher-k3s-common-${2}] -name=Rancher K3s Common (${2}) -baseurl=https://${1}/k3s/${2}/common/${4}/noarch -enabled=1 -gpgcheck=1 -repo_gpgcheck=0 -gpgkey=https://${1}/public.key -EOF - case ${3} in - sle) - rpm_installer="zypper --gpg-auto-import-keys" - if [ "${TRANSACTIONAL_UPDATE=false}" != "true" ] && [ -x /usr/sbin/transactional-update ]; then - rpm_installer="transactional-update --no-selfupdate -d run ${rpm_installer}" - : "${INSTALL_K3S_SKIP_START:=true}" - fi - ;; - *) - rpm_installer="yum" - ;; - esac - if [ "${rpm_installer}" = "yum" ] && [ -x /usr/bin/dnf ]; then - rpm_installer=dnf - fi - # shellcheck disable=SC2086 - $SUDO ${rpm_installer} install -y "k3s-selinux" - fi - return -} - -# --- download and verify k3s --- -download_and_verify() { - if can_skip_download_binary; then - info 'Skipping k3s download and verify' - verify_k3s_is_executable - return - fi - - setup_verify_arch - verify_downloader curl || verify_downloader wget || fatal 'Can not find curl or wget for downloading files' - setup_tmp - get_release_version - download_hash - - if installed_hash_matches; then - info 'Skipping binary downloaded, installed k3s matches hash' - return - fi - - download_binary - verify_binary - setup_binary -} - -# --- add additional utility links --- -create_symlinks() { - [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return - [ "${INSTALL_K3S_SYMLINK}" = skip ] && return - - for cmd in kubectl crictl ctr; do - if [ ! -e ${BIN_DIR}/${cmd} ] || [ "${INSTALL_K3S_SYMLINK}" = force ]; then - which_cmd=$(command -v ${cmd} 2>/dev/null || true) - if [ -z "${which_cmd}" ] || [ "${INSTALL_K3S_SYMLINK}" = force ]; then - info "Creating ${BIN_DIR}/${cmd} symlink to k3s" - $SUDO ln -sf k3s ${BIN_DIR}/${cmd} - else - info "Skipping ${BIN_DIR}/${cmd} symlink to k3s, command exists in PATH at ${which_cmd}" - fi - else - info "Skipping ${BIN_DIR}/${cmd} symlink to k3s, already exists" - fi - done -} - -# --- create killall script --- -create_killall() { - [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return - info "Creating killall script ${KILLALL_K3S_SH}" - $SUDO tee ${KILLALL_K3S_SH} >/dev/null << \EOF -#!/bin/sh -[ $(id -u) -eq 0 ] || exec sudo $0 $@ - -for bin in /var/lib/rancher/k3s/data/**/bin/; do - [ -d $bin ] && export PATH=$PATH:$bin:$bin/aux -done - -set -x - -for service in /etc/systemd/system/k3s*.service; do - [ -s $service ] && systemctl stop $(basename $service) -done - -for service in /etc/init.d/k3s*; do - [ -x $service ] && $service stop -done - -pschildren() { - ps -e -o ppid= -o pid= | \ - sed -e 's/^\s*//g; s/\s\s*/\t/g;' | \ - grep -w "^$1" | \ - cut -f2 -} - -pstree() { - for pid in $@; do - echo $pid - for child in $(pschildren $pid); do - pstree $child - done - done -} - -killtree() { - kill -9 $( - { set +x; } 2>/dev/null; - pstree $@; - set -x; - ) 2>/dev/null -} - -getshims() { - ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1 -} - -killtree $({ set +x; } 2>/dev/null; getshims; set -x) - -do_unmount_and_remove() { - set +x - while read -r _ path _; do - case "$path" in $1*) echo "$path" ;; esac - done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount "$0" && rm -rf "$0"' - set -x -} - -do_unmount_and_remove '/run/k3s' -do_unmount_and_remove '/var/lib/rancher/k3s' -do_unmount_and_remove '/var/lib/kubelet/pods' -do_unmount_and_remove '/var/lib/kubelet/plugins' -do_unmount_and_remove '/run/netns/cni-' - -# Remove CNI namespaces -ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete - -# Delete network interface(s) that match 'master cni0' -ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do - iface=${iface%%@*} - [ -z "$iface" ] || ip link delete $iface -done -ip link delete cni0 -ip link delete flannel.1 -ip link delete flannel-v6.1 -ip link delete kube-ipvs0 -ip link delete flannel-wg -ip link delete flannel-wg-v6 -rm -rf /var/lib/cni/ -iptables-save | grep -v KUBE- | grep -v CNI- | grep -v flannel | iptables-restore -ip6tables-save | grep -v KUBE- | grep -v CNI- | grep -v flannel | ip6tables-restore -EOF - $SUDO chmod 755 ${KILLALL_K3S_SH} - $SUDO chown root:root ${KILLALL_K3S_SH} -} - -# --- create uninstall script --- -create_uninstall() { - [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = true ] && return - info "Creating uninstall script ${UNINSTALL_K3S_SH}" - $SUDO tee ${UNINSTALL_K3S_SH} >/dev/null << EOF -#!/bin/sh -set -x -[ \$(id -u) -eq 0 ] || exec sudo \$0 \$@ - -${KILLALL_K3S_SH} - -if command -v systemctl; then - systemctl disable ${SYSTEM_NAME} - systemctl reset-failed ${SYSTEM_NAME} - systemctl daemon-reload -fi -if command -v rc-update; then - rc-update delete ${SYSTEM_NAME} default -fi - -rm -f ${FILE_K3S_SERVICE} -rm -f ${FILE_K3S_ENV} - -remove_uninstall() { - rm -f ${UNINSTALL_K3S_SH} -} -trap remove_uninstall EXIT - -if (ls ${SYSTEMD_DIR}/k3s*.service || ls /etc/init.d/k3s*) >/dev/null 2>&1; then - set +x; echo 'Additional k3s services installed, skipping uninstall of k3s'; set -x - exit -fi - -for cmd in kubectl crictl ctr; do - if [ -L ${BIN_DIR}/\$cmd ]; then - rm -f ${BIN_DIR}/\$cmd - fi -done - -rm -rf /etc/rancher/k3s -rm -rf /run/k3s -rm -rf /run/flannel -rm -rf /var/lib/rancher/k3s -rm -rf /var/lib/kubelet -rm -f ${BIN_DIR}/k3s -rm -f ${KILLALL_K3S_SH} - -if type yum >/dev/null 2>&1; then - yum remove -y k3s-selinux - rm -f /etc/yum.repos.d/rancher-k3s-common*.repo -elif type zypper >/dev/null 2>&1; then - uninstall_cmd="zypper remove -y k3s-selinux" - if [ "\${TRANSACTIONAL_UPDATE=false}" != "true" ] && [ -x /usr/sbin/transactional-update ]; then - uninstall_cmd="transactional-update --no-selfupdate -d run \$uninstall_cmd" - fi - \$uninstall_cmd - rm -f /etc/zypp/repos.d/rancher-k3s-common*.repo -fi -EOF - $SUDO chmod 755 ${UNINSTALL_K3S_SH} - $SUDO chown root:root ${UNINSTALL_K3S_SH} -} - -# --- disable current service if loaded -- -systemd_disable() { - $SUDO systemctl disable ${SYSTEM_NAME} >/dev/null 2>&1 || true - $SUDO rm -f /etc/systemd/system/${SERVICE_K3S} || true - $SUDO rm -f /etc/systemd/system/${SERVICE_K3S}.env || true -} - -# --- capture current env and create file containing k3s_ variables --- -create_env_file() { - info "env: Creating environment file ${FILE_K3S_ENV}" - $SUDO touch ${FILE_K3S_ENV} - $SUDO chmod 0600 ${FILE_K3S_ENV} - sh -c export | while read x v; do echo $v; done | grep -E '^(K3S|CONTAINERD)_' | $SUDO tee ${FILE_K3S_ENV} >/dev/null - sh -c export | while read x v; do echo $v; done | grep -Ei '^(NO|HTTP|HTTPS)_PROXY' | $SUDO tee -a ${FILE_K3S_ENV} >/dev/null -} - -# --- write systemd service file --- -create_systemd_service_file() { - info "systemd: Creating service file ${FILE_K3S_SERVICE}" - $SUDO tee ${FILE_K3S_SERVICE} >/dev/null << EOF -[Unit] -Description=Lightweight Kubernetes -Documentation=https://k3s.io -Wants=network-online.target -After=network-online.target - -[Install] -WantedBy=multi-user.target - -[Service] -Type=${SYSTEMD_TYPE} -EnvironmentFile=-/etc/default/%N -EnvironmentFile=-/etc/sysconfig/%N -EnvironmentFile=-${FILE_K3S_ENV} -KillMode=process -Delegate=yes -# Having non-zero Limit*s causes performance problems due to accounting overhead -# in the kernel. We recommend using cgroups to do container-local accounting. -LimitNOFILE=1048576 -LimitNPROC=infinity -LimitCORE=infinity -TasksMax=infinity -TimeoutStartSec=0 -Restart=always -RestartSec=5s -ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service' -ExecStartPre=-/sbin/modprobe br_netfilter -ExecStartPre=-/sbin/modprobe overlay -ExecStart=${BIN_DIR}/k3s \\ - ${CMD_K3S_EXEC} - -EOF -} - -# --- write openrc service file --- -create_openrc_service_file() { - LOG_FILE=/var/log/${SYSTEM_NAME}.log - - info "openrc: Creating service file ${FILE_K3S_SERVICE}" - $SUDO tee ${FILE_K3S_SERVICE} >/dev/null << EOF -#!/sbin/openrc-run - -depend() { - after network-online - want cgroups -} - -start_pre() { - rm -f /tmp/k3s.* -} - -supervisor=supervise-daemon -name=${SYSTEM_NAME} -command="${BIN_DIR}/k3s" -command_args="$(escape_dq "${CMD_K3S_EXEC}") - >>${LOG_FILE} 2>&1" - -output_log=${LOG_FILE} -error_log=${LOG_FILE} - -pidfile="/var/run/${SYSTEM_NAME}.pid" -respawn_delay=5 -respawn_max=0 - -set -o allexport -if [ -f /etc/environment ]; then source /etc/environment; fi -if [ -f ${FILE_K3S_ENV} ]; then source ${FILE_K3S_ENV}; fi -set +o allexport -EOF - $SUDO chmod 0755 ${FILE_K3S_SERVICE} - - $SUDO tee /etc/logrotate.d/${SYSTEM_NAME} >/dev/null << EOF -${LOG_FILE} { - missingok - notifempty - copytruncate -} -EOF -} - -# --- write systemd or openrc service file --- -create_service_file() { - [ "${HAS_SYSTEMD}" = true ] && create_systemd_service_file - [ "${HAS_OPENRC}" = true ] && create_openrc_service_file - return 0 -} - -# --- get hashes of the current k3s bin and service files -get_installed_hashes() { - $SUDO sha256sum ${BIN_DIR}/k3s ${FILE_K3S_SERVICE} ${FILE_K3S_ENV} 2>&1 || true -} - -# --- enable and start systemd service --- -systemd_enable() { - info "systemd: Enabling ${SYSTEM_NAME} unit" - $SUDO systemctl enable ${FILE_K3S_SERVICE} >/dev/null - $SUDO systemctl daemon-reload >/dev/null -} - -systemd_start() { - info "systemd: Starting ${SYSTEM_NAME}" - $SUDO systemctl restart ${SYSTEM_NAME} -} - -# --- enable and start openrc service --- -openrc_enable() { - info "openrc: Enabling ${SYSTEM_NAME} service for default runlevel" - $SUDO rc-update add ${SYSTEM_NAME} default >/dev/null -} - -openrc_start() { - info "openrc: Starting ${SYSTEM_NAME}" - $SUDO ${FILE_K3S_SERVICE} restart -} - -# --- startup systemd or openrc service --- -service_enable_and_start() { - if [ -f "/proc/cgroups" ] && [ "$(grep memory /proc/cgroups | while read -r n n n enabled; do echo $enabled; done)" -eq 0 ]; - then - info 'Failed to find memory cgroup, you may need to add "cgroup_memory=1 cgroup_enable=memory" to your linux cmdline (/boot/cmdline.txt on a Raspberry Pi)' - fi - - [ "${INSTALL_K3S_SKIP_ENABLE}" = true ] && return - - [ "${HAS_SYSTEMD}" = true ] && systemd_enable - [ "${HAS_OPENRC}" = true ] && openrc_enable - - [ "${INSTALL_K3S_SKIP_START}" = true ] && return - - POST_INSTALL_HASHES=$(get_installed_hashes) - if [ "${PRE_INSTALL_HASHES}" = "${POST_INSTALL_HASHES}" ] && [ "${INSTALL_K3S_FORCE_RESTART}" != true ]; then - info 'No change detected so skipping service start' - return - fi - - [ "${HAS_SYSTEMD}" = true ] && systemd_start - [ "${HAS_OPENRC}" = true ] && openrc_start - return 0 -} - -# --- re-evaluate args to include env command --- -eval set -- $(escape "${INSTALL_K3S_EXEC}") $(quote "$@") - -# --- run the install process -- -{ - verify_system - setup_env "$@" - download_and_verify - setup_selinux - create_symlinks - create_killall - create_uninstall - systemd_disable - create_env_file - create_service_file - service_enable_and_start -} \ No newline at end of file diff --git a/pkg/service/binary/kubernetes/binary.go b/pkg/service/binary/kubernetes/binary.go deleted file mode 100644 index ec17eb1f..00000000 --- a/pkg/service/binary/kubernetes/binary.go +++ /dev/null @@ -1,220 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "embed" - "path/filepath" - "text/template" - "time" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" - "github.com/kubesphere/kubekey/v3/pkg/service/operation" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/file" - "github.com/kubesphere/kubekey/v3/pkg/service/util" -) - -//go:embed templates -var f embed.FS - -// Download downloads binaries. -func (s *Service) Download(timeout time.Duration) error { - if err := s.DownloadAll(timeout); err != nil { - return err - } - - if err := s.ConfigureKubelet(); err != nil { - return err - } - - if err := s.ConfigureKubeadm(); err != nil { - return err - } - - return nil -} - -// DownloadAll downloads all binaries. -func (s *Service) DownloadAll(timeout time.Duration) error { - kubeadm, err := s.getKubeadmService(s.instanceScope.KubernetesVersion(), s.instanceScope.Arch()) - if err != nil { - return err - } - kubelet, err := s.getKubeletService(s.instanceScope.KubernetesVersion(), s.instanceScope.Arch()) - if err != nil { - return err - } - kubecni, err := s.getKubecniService(file.KubecniDefaultVersion, s.instanceScope.Arch()) - if err != nil { - return err - } - kubectl, err := s.getKubectlService(s.instanceScope.KubernetesVersion(), s.instanceScope.Arch()) - if err != nil { - return err - } - - binaries := []operation.Binary{ - kubeadm, - kubelet, - kubecni, - kubectl, - } - - zone := s.scope.ComponentZone() - host := s.scope.ComponentHost() - overrideMap := make(map[string]infrav1.Override) - for _, o := range s.scope.ComponentOverrides() { - overrideMap[o.ID+o.Version+o.Arch] = o - } - - for _, b := range binaries { - override := overrideMap[b.ID()+b.Version()+b.Arch()] - if err := util.DownloadAndCopy(s.instanceScope, b, zone, host, override.Path, override.URL, override.Checksum.Value, timeout); err != nil { - return err - } - if err := b.Chmod("+x"); err != nil { - return err - } - } - - if _, err := s.sshClient.SudoCmdf("tar Cxzvf %s %s", filepath.Dir(kubecni.RemotePath()), kubecni.RemotePath()); err != nil { - return err - } - - return nil -} - -// ConfigureKubelet configures kubelet. -func (s *Service) ConfigureKubelet() error { - kubelet, err := s.getKubeletService(s.instanceScope.KubernetesVersion(), s.instanceScope.Arch()) - if err != nil { - return err - } - - if _, err := s.sshClient.SudoCmdf("ln -snf %s /usr/bin/kubelet", kubelet.RemotePath()); err != nil { - return err - } - - temp, err := template.ParseFS(f, "templates/kubelet.service") - if err != nil { - return err - } - - svc, err := s.getTemplateService( - temp, - nil, - filepath.Join(file.SystemdDir, temp.Name())) - if err != nil { - return err - } - if err := svc.RenderToLocal(); err != nil { - return err - } - if err := svc.Copy(true); err != nil { - return err - } - - env, err := template.ParseFS(f, "templates/kubelet.conf") - if err != nil { - return err - } - - envSvc, err := s.getTemplateService( - env, - file.Data{ - "NodeIP": s.instanceScope.InternalAddress(), - "Hostname": s.instanceScope.HostName(), - }, - filepath.Join(file.SystemdDir, "kubelet.service.d", env.Name()), - ) - if err != nil { - return err - } - if err := envSvc.RenderToLocal(); err != nil { - return err - } - if err := envSvc.Copy(true); err != nil { - return err - } - - if _, err := s.sshClient.SudoCmdf("systemctl disable kubelet && systemctl enable kubelet"); err != nil { - return err - } - return nil -} - -// ConfigureKubeadm configures kubeadm. -func (s *Service) ConfigureKubeadm() error { - kubeadm, err := s.getKubeadmService(s.instanceScope.KubernetesVersion(), s.instanceScope.Arch()) - if err != nil { - return err - } - - if _, err := s.sshClient.SudoCmdf("ln -snf %s /usr/bin/kubeadm", kubeadm.RemotePath()); err != nil { - return err - } - return nil -} - -// UpgradeDownload downloads upgrade binaries. -func (s *Service) UpgradeDownload(timeout time.Duration) error { - return s.downloadUpgradeBinaries(timeout) -} - -// downloadUpgradeBinaries downloads kubeadm, kubelet and kubectl. -func (s *Service) downloadUpgradeBinaries(timeout time.Duration) error { - kubeadm, err := s.getKubeadmService(s.instanceScope.InPlaceUpgradeVersion(), s.instanceScope.Arch()) - if err != nil { - return err - } - kubelet, err := s.getKubeletService(s.instanceScope.InPlaceUpgradeVersion(), s.instanceScope.Arch()) - if err != nil { - return err - } - kubectl, err := s.getKubectlService(s.instanceScope.InPlaceUpgradeVersion(), s.instanceScope.Arch()) - if err != nil { - return err - } - - binaries := []operation.Binary{ - kubeadm, - kubelet, - kubectl, - } - - zone := s.scope.ComponentZone() - host := s.scope.ComponentHost() - overrideMap := make(map[string]infrav1.Override) - for _, o := range s.scope.ComponentOverrides() { - overrideMap[o.ID+o.Version+o.Arch] = o - } - - for _, b := range binaries { - override := overrideMap[b.ID()+b.Version()+b.Arch()] - if err := util.DownloadAndCopy(s.instanceScope, b, zone, host, override.Path, override.URL, override.Checksum.Value, timeout); err != nil { - return err - } - if err := b.Chmod("+x"); err != nil { - return err - } - if _, err := s.sshClient.SudoCmdf("ln -snf %s /usr/bin/%s", b.RemotePath(), b.Name()); err != nil { - return err - } - } - - return nil -} diff --git a/pkg/service/binary/kubernetes/doc.go b/pkg/service/binary/kubernetes/doc.go deleted file mode 100644 index eae16622..00000000 --- a/pkg/service/binary/kubernetes/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package kubernetes define the binaries operations on the remote instance. -package kubernetes diff --git a/pkg/service/binary/kubernetes/service.go b/pkg/service/binary/kubernetes/service.go deleted file mode 100644 index 4c649a31..00000000 --- a/pkg/service/binary/kubernetes/service.go +++ /dev/null @@ -1,84 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package kubernetes - -import ( - "text/template" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/scope" - "github.com/kubesphere/kubekey/v3/pkg/service/operation" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/file" -) - -// Service holds a collection of interfaces. -// The interfaces are broken down like this to group functions together. -type Service struct { - sshClient ssh.Interface - scope scope.KKInstanceScope - instanceScope *scope.InstanceScope - - templateFactory func(sshClient ssh.Interface, template *template.Template, data file.Data, dst string) (operation.Template, error) - kubeadmFactory func(sshClient ssh.Interface, version, arch string) (operation.Binary, error) - kubeletFactory func(sshClient ssh.Interface, version, arch string) (operation.Binary, error) - kubecniFactory func(sshClient ssh.Interface, version, arch string) (operation.Binary, error) - kubectlFactory func(sshClient ssh.Interface, version, arch string) (operation.Binary, error) -} - -// NewService returns a new service given the remote instance. -func NewService(sshClient ssh.Interface, scope scope.KKInstanceScope, instanceScope *scope.InstanceScope) *Service { - return &Service{ - sshClient: sshClient, - scope: scope, - instanceScope: instanceScope, - } -} - -func (s *Service) getTemplateService(template *template.Template, data file.Data, dst string) (operation.Template, error) { - if s.templateFactory != nil { - return s.templateFactory(s.sshClient, template, data, dst) - } - return file.NewTemplate(s.sshClient, s.scope.RootFs(), template, data, dst) -} - -func (s *Service) getKubeadmService(version, arch string) (operation.Binary, error) { - if s.kubeadmFactory != nil { - return s.kubeadmFactory(s.sshClient, version, arch) - } - return file.NewKubeadm(s.sshClient, s.scope.RootFs(), version, arch) -} - -func (s *Service) getKubeletService(version, arch string) (operation.Binary, error) { - if s.kubeletFactory != nil { - return s.kubeletFactory(s.sshClient, version, arch) - } - return file.NewKubelet(s.sshClient, s.scope.RootFs(), version, arch) -} - -func (s *Service) getKubecniService(version, arch string) (operation.Binary, error) { - if s.kubecniFactory != nil { - return s.kubecniFactory(s.sshClient, version, arch) - } - return file.NewKubecni(s.sshClient, s.scope.RootFs(), version, arch) -} - -func (s *Service) getKubectlService(version, arch string) (operation.Binary, error) { - if s.kubectlFactory != nil { - return s.kubectlFactory(s.sshClient, version, arch) - } - return file.NewKubectl(s.sshClient, s.scope.RootFs(), version, arch) -} diff --git a/pkg/service/binary/kubernetes/templates/kubelet.conf b/pkg/service/binary/kubernetes/templates/kubelet.conf deleted file mode 100644 index b3d6cbf3..00000000 --- a/pkg/service/binary/kubernetes/templates/kubelet.conf +++ /dev/null @@ -1,12 +0,0 @@ -# Note: This dropin only works with kubeadm and kubelet v1.11+ -[Service] -Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" -Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" -# This is a file that "kubeadm init" and "kubeadm join" generate at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically -EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env -# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use -# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file. -EnvironmentFile=-/etc/default/kubelet -Environment="KUBELET_EXTRA_ARGS=--node-ip={{ .NodeIP }} --hostname-override={{ .Hostname }}" -ExecStart= -ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS diff --git a/pkg/service/binary/kubernetes/templates/kubelet.service b/pkg/service/binary/kubernetes/templates/kubelet.service deleted file mode 100644 index bcd4bd91..00000000 --- a/pkg/service/binary/kubernetes/templates/kubelet.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=kubelet: The Kubernetes Node Agent -Documentation=http://kubernetes.io/docs/ - -[Service] -ExecStart=/usr/local/bin/kubelet -Restart=always -RestartSec=10s - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/service/bootstrap/bootstrap.go b/pkg/service/bootstrap/bootstrap.go deleted file mode 100644 index 176547da..00000000 --- a/pkg/service/bootstrap/bootstrap.go +++ /dev/null @@ -1,233 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package bootstrap - -import ( - "embed" - "fmt" - "os" - "path/filepath" - "text/template" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/pkg/service/operation/directory" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/file" - "github.com/kubesphere/kubekey/v3/pkg/util/filesystem" -) - -//go:embed templates -var f embed.FS - -// AddUsers adds a kube user to the Linux. -func (s *Service) AddUsers() error { - userService := s.getUserService("kube", "Kubernetes user") - - // todo: if need to create a etcd user - return userService.Add() -} - -// SetHostname sets the hostname of the machine. -func (s *Service) SetHostname() error { - if _, err := s.sshClient.SudoCmdf( - "hostnamectl set-hostname %s && sed -i '/^127.0.1.1/s/.*/127.0.1.1 %s/g' /etc/hosts", - s.instanceScope.HostName(), - s.instanceScope.HostName()); err != nil { - return errors.Wrapf(err, "failed to set host name [%s]", s.instanceScope.HostName()) - } - return nil -} - -// CreateDirectory creates some common directories. -func (s *Service) CreateDirectory() error { - makeDirs := []string{ - directory.BinDir, - directory.KubeConfigDir, - directory.KubeCertDir, - directory.KubeManifestDir, - directory.KubeScriptDir, - directory.KubeletFlexvolumesPluginsDir, - "/var/lib/calico", - "/etc/cni/net.d", - "/opt/cni/bin", - } - for _, dir := range makeDirs { - dirService := s.getDirectoryService(dir, os.FileMode(filesystem.FileMode0755)) - if err := dirService.Make(); err != nil { - return err - } - } - - chownDirs := []string{ - directory.KubeConfigDir, - directory.KubeCertDir, - directory.KubeManifestDir, - directory.KubeScriptDir, - "/usr/libexec/kubernetes", - "/etc/cni", - "/opt/cni", - "/var/lib/calico", - } - for _, dir := range chownDirs { - dirService := s.getDirectoryService(dir, os.FileMode(filesystem.FileMode0755)) - if err := dirService.Chown("kube"); err != nil { - return err - } - } - return nil -} - -// ResetTmpDirectory resets the temporary "/tmp/kubekey" directory. -func (s *Service) ResetTmpDirectory() error { - dirService := s.getDirectoryService(directory.TmpDir, os.FileMode(filesystem.FileMode0777)) - if err := dirService.Remove(); err != nil { - return err - } - if err := dirService.Make(); err != nil { - return err - } - return nil -} - -// ExecInitScript executes the init script on the remote instance. -func (s *Service) ExecInitScript() error { - var ( - hostsList []string - lbHost string - ) - - for _, host := range s.scope.AllInstancesInfo() { - if host.Name != "" { - hostsList = append(hostsList, fmt.Sprintf("%s %s.%s %s", - host.InternalAddress, - host.Name, - s.scope.KubernetesClusterName(), - host.Name)) - } - } - hostsList = append(hostsList, lbHost) - - temp, err := template.ParseFS(f, "templates/initOS.sh") - if err != nil { - return err - } - - svc, err := s.getTemplateService( - temp, - file.Data{ - "Hosts": hostsList, - }, - filepath.Join(directory.KubeScriptDir, temp.Name())) - if err != nil { - return err - } - if err := svc.RenderToLocal(); err != nil { - return err - } - if err := svc.Copy(true); err != nil { - return err - } - if err := svc.Chmod("+x"); err != nil { - return err - } - if _, err := s.sshClient.SudoCmd(svc.RemotePath()); err != nil { - return err - } - return nil -} - -// KubeadmReset resets the Kubernetes by using kubeadm. -func (s *Service) KubeadmReset(criSocket string) error { - cmd := "kubeadm reset -f" - if criSocket != "" { - cmd = cmd + " --cri-socket " + criSocket - } - _, _ = s.sshClient.SudoCmd(cmd) - return nil -} - -// ResetNetwork resets the network configuration. -func (s *Service) ResetNetwork() error { - networkResetCmds := []string{ - "iptables -F", - "iptables -X", - "iptables -F -t nat", - "iptables -X -t nat", - "ipvsadm -C", - "ip link del kube-ipvs0", - "ip link del nodelocaldns", - "ip link del cni0", - "ip link del flannel.1", - "ip link del flannel-v6.1", - "ip link del flannel-wg", - "ip link del flannel-wg-v6", - "ip link del cilium_host", - "ip link del cilium_vxlan", - "ip -br link show | grep 'cali[a-f0-9]*' | awk -F '@' '{print $1}' | xargs -r -t -n 1 ip link del", - "ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns del", - } - for _, cmd := range networkResetCmds { - _, _ = s.sshClient.SudoCmd(cmd) - } - return nil -} - -// RemoveFiles removes some directories and files that may have been created by the Kubernetes and other related components. -func (s *Service) RemoveFiles() error { - removeDirs := []string{ - directory.KubeConfigDir, - directory.KubeScriptDir, - "/var/log/calico", - "/etc/cni", - "/var/log/pods/", - "/var/lib/cni", - "/var/lib/calico", - "/var/lib/kubelet", - "/var/lib/rook", - "/run/calico", - "/run/flannel", - "/etc/flannel", - "/var/openebs", - "/etc/systemd/system/kubelet.service", - "/etc/systemd/system/kubelet.service.d", - "/tmp/kubekey", - "/etc/kubekey", - "/run/kubeadm/*", - } - for _, dir := range removeDirs { - dirService := s.getDirectoryService(dir, 0) - _ = dirService.Remove() - } - return nil -} - -// DaemonReload reloads the systemd daemon and restart the containerd. -func (s *Service) DaemonReload() error { - _, _ = s.sshClient.SudoCmd("systemctl daemon-reload && exit 0") - _, _ = s.sshClient.SudoCmd("systemctl restart containerd") - return nil -} - -// UninstallK3s uninstalls the k3s. -func (s *Service) UninstallK3s() error { - if s.instanceScope.IsControlPlane() { - _, _ = s.sshClient.SudoCmd("/usr/local/bin/k3s-uninstall.sh") - } else { - _, _ = s.sshClient.SudoCmd("/usr/local/bin/k3s-agent-uninstall.sh") - } - return nil -} diff --git a/pkg/service/bootstrap/doc.go b/pkg/service/bootstrap/doc.go deleted file mode 100644 index 5cffb94e..00000000 --- a/pkg/service/bootstrap/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package bootstrap defines the CAPKK bootstrap operations on the remote instance. -package bootstrap diff --git a/pkg/service/bootstrap/service.go b/pkg/service/bootstrap/service.go deleted file mode 100644 index 694b5dc1..00000000 --- a/pkg/service/bootstrap/service.go +++ /dev/null @@ -1,71 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package bootstrap - -import ( - "os" - "text/template" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/scope" - "github.com/kubesphere/kubekey/v3/pkg/service/operation" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/directory" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/file" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/user" -) - -// Service holds a collection of interfaces. -// The interfaces are broken down like this to group functions together. -type Service struct { - sshClient ssh.Interface - scope scope.LBScope - instanceScope *scope.InstanceScope - - userFactory func(sshClient ssh.Interface, name, desc string) operation.User - directoryFactory func(sshClient ssh.Interface, path string, mode os.FileMode) operation.Directory - templateFactory func(sshClient ssh.Interface, template *template.Template, data file.Data, dst string) (operation.Template, error) -} - -// NewService returns a new service given the remote instance kubekey build-in bootstrap provision client. -func NewService(sshClient ssh.Interface, scope scope.LBScope, instanceScope *scope.InstanceScope) *Service { - return &Service{ - sshClient: sshClient, - scope: scope, - instanceScope: instanceScope, - } -} - -func (s *Service) getUserService(name, desc string) operation.User { - if s.userFactory != nil { - return s.userFactory(s.sshClient, name, desc) - } - return user.NewService(s.sshClient, name, desc) -} - -func (s *Service) getDirectoryService(path string, mode os.FileMode) operation.Directory { - if s.directoryFactory != nil { - return s.directoryFactory(s.sshClient, path, mode) - } - return directory.NewService(s.sshClient, path, mode) -} - -func (s *Service) getTemplateService(template *template.Template, data file.Data, dst string) (operation.Template, error) { - if s.templateFactory != nil { - return s.templateFactory(s.sshClient, template, data, dst) - } - return file.NewTemplate(s.sshClient, s.scope.RootFs(), template, data, dst) -} diff --git a/pkg/service/bootstrap/templates/initOS.sh b/pkg/service/bootstrap/templates/initOS.sh deleted file mode 100644 index 69cca282..00000000 --- a/pkg/service/bootstrap/templates/initOS.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2020 The KubeSphere Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -swapoff -a -sed -i /^[^#]*swap*/s/^/\#/g /etc/fstab - -# See https://github.com/kubernetes/website/issues/14457 -if [ -f /etc/selinux/config ]; then - sed -ri 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config -fi -# for ubuntu: sudo apt install selinux-utils -# for centos: yum install selinux-policy -if command -v setenforce &> /dev/null -then - setenforce 0 - getenforce -fi - -echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf -echo 'net.bridge.bridge-nf-call-arptables = 1' >> /etc/sysctl.conf -echo 'net.bridge.bridge-nf-call-ip6tables = 1' >> /etc/sysctl.conf -echo 'net.bridge.bridge-nf-call-iptables = 1' >> /etc/sysctl.conf -echo 'net.ipv4.ip_local_reserved_ports = 30000-32767' >> /etc/sysctl.conf -echo 'vm.max_map_count = 262144' >> /etc/sysctl.conf -echo 'vm.swappiness = 1' >> /etc/sysctl.conf -echo 'fs.inotify.max_user_instances = 524288' >> /etc/sysctl.conf -echo 'kernel.pid_max = 65535' >> /etc/sysctl.conf - - -#See https://imroc.io/posts/kubernetes/troubleshooting-with-kubernetes-network/ -sed -r -i "s@#{0,}?net.ipv4.tcp_tw_recycle ?= ?(0|1)@net.ipv4.tcp_tw_recycle = 0@g" /etc/sysctl.conf - -sed -r -i "s@#{0,}?net.ipv4.ip_forward ?= ?(0|1)@net.ipv4.ip_forward = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-arptables ?= ?(0|1)@net.bridge.bridge-nf-call-arptables = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-ip6tables ?= ?(0|1)@net.bridge.bridge-nf-call-ip6tables = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-iptables ?= ?(0|1)@net.bridge.bridge-nf-call-iptables = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?net.ipv4.ip_local_reserved_ports ?= ?([0-9]{1,}-{0,1},{0,1}){1,}@net.ipv4.ip_local_reserved_ports = 30000-32767@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?vm.max_map_count ?= ?([0-9]{1,})@vm.max_map_count = 262144@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?vm.swappiness ?= ?([0-9]{1,})@vm.swappiness = 1@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?fs.inotify.max_user_instances ?= ?([0-9]{1,})@fs.inotify.max_user_instances = 524288@g" /etc/sysctl.conf -sed -r -i "s@#{0,}?kernel.pid_max ?= ?([0-9]{1,})@kernel.pid_max = 65535@g" /etc/sysctl.conf - -awk ' !x[$0]++{print > "/etc/sysctl.conf"}' /etc/sysctl.conf - -systemctl stop firewalld 1>/dev/null 2>/dev/null -systemctl disable firewalld 1>/dev/null 2>/dev/null -systemctl stop ufw 1>/dev/null 2>/dev/null -systemctl disable ufw 1>/dev/null 2>/dev/null - -modinfo br_netfilter > /dev/null 2>&1 -if [ $? -eq 0 ]; then - modprobe br_netfilter - mkdir -p /etc/modules-load.d - echo 'br_netfilter' > /etc/modules-load.d/kubekey-br_netfilter.conf -fi - -modinfo overlay > /dev/null 2>&1 -if [ $? -eq 0 ]; then - modprobe overlay - echo 'overlay' >> /etc/modules-load.d/kubekey-br_netfilter.conf -fi - -modprobe ip_vs -modprobe ip_vs_rr -modprobe ip_vs_wrr -modprobe ip_vs_sh - -cat > /etc/modules-load.d/kube_proxy-ipvs.conf << EOF -ip_vs -ip_vs_rr -ip_vs_wrr -ip_vs_sh -EOF - -modprobe nf_conntrack_ipv4 1>/dev/null 2>/dev/null -if [ $? -eq 0 ]; then - echo 'nf_conntrack_ipv4' > /etc/modules-load.d/kube_proxy-ipvs.conf -else - modprobe nf_conntrack - echo 'nf_conntrack' > /etc/modules-load.d/kube_proxy-ipvs.conf -fi -sysctl -p - -sed -i ':a;$!{N;ba};s@# kubekey hosts BEGIN.*# kubekey hosts END@@' /etc/hosts -sed -i '/^$/N;/\n$/N;//D' /etc/hosts - -cat >>/etc/hosts< /proc/sys/vm/drop_caches - -# Make sure the iptables utility doesn't use the nftables backend. -update-alternatives --set iptables /usr/sbin/iptables-legacy >/dev/null 2>&1 || true -update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy >/dev/null 2>&1 || true -update-alternatives --set arptables /usr/sbin/arptables-legacy >/dev/null 2>&1 || true -update-alternatives --set ebtables /usr/sbin/ebtables-legacy >/dev/null 2>&1 || true - -ulimit -u 65535 -ulimit -n 65535 diff --git a/pkg/service/containermanager/containerd.go b/pkg/service/containermanager/containerd.go deleted file mode 100644 index ab2563b8..00000000 --- a/pkg/service/containermanager/containerd.go +++ /dev/null @@ -1,366 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package containermanager - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "text/template" - "time" - - "github.com/pkg/errors" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/scope" - "github.com/kubesphere/kubekey/v3/pkg/service/operation" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/file" - "github.com/kubesphere/kubekey/v3/pkg/service/util" -) - -// ContainerdService is a ContainerManager service implementation for containerd. -type ContainerdService struct { - sshClient ssh.Interface - - scope scope.KKInstanceScope - instanceScope *scope.InstanceScope - - templateFactory func(sshClient ssh.Interface, template *template.Template, data file.Data, dst string) (operation.Template, error) - runcFactory func(sshClient ssh.Interface, version, arch string) (operation.Binary, error) - containerdFactory func(sshClient ssh.Interface, version, arch string) (operation.Binary, error) - crictlFactory func(sshClient ssh.Interface, version, arch string) (operation.Binary, error) -} - -// NewContainerdService returns a new ContainerdService given the remote instance container manager client. -func NewContainerdService(sshClient ssh.Interface, scope scope.KKInstanceScope, instanceScope *scope.InstanceScope) *ContainerdService { - return &ContainerdService{ - sshClient: sshClient, - scope: scope, - instanceScope: instanceScope, - } -} - -func (s *ContainerdService) getRuncService(sshClient ssh.Interface, version, arch string) (operation.Binary, error) { - if s.runcFactory != nil { - return s.runcFactory(sshClient, version, arch) - } - return file.NewRunc(sshClient, s.scope.RootFs(), version, arch) -} - -func (s *ContainerdService) getContainerdService(sshClient ssh.Interface, version, arch string) (operation.Binary, error) { - if s.containerdFactory != nil { - return s.containerdFactory(sshClient, version, arch) - } - return file.NewContainerd(sshClient, s.scope.RootFs(), version, arch) -} - -func (s *ContainerdService) getCrictlService(sshClient ssh.Interface, version, arch string) (operation.Binary, error) { - if s.crictlFactory != nil { - return s.crictlFactory(sshClient, version, arch) - } - return file.NewCrictl(sshClient, s.scope.RootFs(), version, arch) -} - -func (s *ContainerdService) getTemplateService(template *template.Template, data file.Data, dst string) (operation.Template, error) { - if s.templateFactory != nil { - return s.templateFactory(s.sshClient, template, data, dst) - } - return file.NewTemplate(s.sshClient, s.scope.RootFs(), template, data, dst) -} - -// Type returns the type containerd of the container manager. -func (s *ContainerdService) Type() string { - return file.ContainerdID -} - -// Version returns the version of the container manager. -func (s *ContainerdService) Version() string { - return s.instanceScope.KKInstance.Spec.ContainerManager.Version -} - -// IsExist returns true if the container manager is installed. -func (s *ContainerdService) IsExist() bool { - res, err := s.sshClient.SudoCmd( - "if [ -z $(which containerd) ] || [ ! -e /run/containerd/containerd.sock ]; " + - "then echo 'not exist'; " + - "fi") - if err != nil { - return false - } - if strings.Contains(res, "not exist") { - return false - } - return true -} - -// Get gets the binary of containerd and related components and copy them to the remote instance. -func (s *ContainerdService) Get(timeout time.Duration) error { - containerd, err := s.getContainerdService(s.sshClient, s.Version(), s.instanceScope.Arch()) - if err != nil { - return err - } - runc, err := s.getRuncService(s.sshClient, file.RuncDefaultVersion, s.instanceScope.Arch()) - if err != nil { - return err - } - crictl, err := s.getCrictlService(s.sshClient, s.instanceScope.KKInstance.Spec.ContainerManager.CRICTLVersion, s.instanceScope.Arch()) - if err != nil { - return err - } - - binaries := []operation.Binary{ - containerd, - runc, - crictl, - } - - zone := s.scope.ComponentZone() - host := s.scope.ComponentHost() - overrideMap := make(map[string]infrav1.Override) - for _, o := range s.scope.ComponentOverrides() { - overrideMap[o.ID+o.Version+o.Arch] = o - } - - for _, b := range binaries { - override := overrideMap[b.ID()+b.Version()+b.Arch()] - if err := util.DownloadAndCopy(s.instanceScope, b, zone, host, override.Path, override.URL, override.Checksum.Value, timeout); err != nil { - return err - } - } - - // /usr/local - dir := filepath.Dir(filepath.Dir(containerd.RemotePath())) - if _, err := s.sshClient.SudoCmdf("tar Cxzvf %s %s", dir, containerd.RemotePath()); err != nil { - return err - } - if _, err := s.sshClient.SudoCmdf("tar Cxzvf %s %s", filepath.Dir(crictl.RemotePath()), crictl.RemotePath()); err != nil { - return err - } - return nil -} - -// Install installs the container manager and related components. -func (s *ContainerdService) Install() error { - if err := s.installContainerd(); err != nil { - return err - } - if err := s.installRunc(); err != nil { - return err - } - if err := s.installCrictl(); err != nil { - return err - } - return nil -} - -func (s *ContainerdService) installContainerd() error { - if err := s.generateContainerdConfig(); err != nil { - return err - } - if err := s.generateContainerdService(); err != nil { - return err - } - if _, err := s.sshClient.SudoCmd("systemctl daemon-reload && systemctl enable containerd && systemctl start containerd"); err != nil { - return err - } - return nil -} - -func (s *ContainerdService) generateContainerdService() error { - temp, err := template.ParseFS(f, "templates/containerd.service") - if err != nil { - return err - } - - svc, err := s.getTemplateService(temp, nil, filepath.Join(file.SystemdDir, temp.Name())) - if err != nil { - return err - } - if err := svc.RenderToLocal(); err != nil { - return err - } - if err := svc.Copy(true); err != nil { - return err - } - return nil -} - -func (s *ContainerdService) generateContainerdConfig() error { - temp, err := template.ParseFS(f, "templates/config.toml") - if err != nil { - return err - } - - svc, err := s.getTemplateService( - temp, - file.Data{ - "Mirrors": s.mirrors(), - "InsecureRegistries": s.insecureRegistry(), - // todo: handle sandbox image - // "SandBoxImage": images.GetImage(m.Runtime, m.KubeConf, "pause").ImageName(), - "PrivateRegistry": s.privateRegistry(), - "Auth": s.auth(), - }, - filepath.Join("/etc/containerd/", temp.Name())) - if err != nil { - return err - } - if err := svc.RenderToLocal(); err != nil { - return err - } - if err := svc.Copy(true); err != nil { - return err - } - return nil -} - -func (s *ContainerdService) mirrors() string { - var m string - if s.scope.GlobalRegistry() != nil { - var mirrorsArr []string - for _, mirror := range s.scope.GlobalRegistry().RegistryMirrors { - mirrorsArr = append(mirrorsArr, fmt.Sprintf("%q", mirror)) - } - m = strings.Join(mirrorsArr, ", ") - } - return m -} - -func (s *ContainerdService) insecureRegistry() []string { - var insecureRegistries []string - if s.scope.GlobalRegistry() != nil { - insecureRegistries = s.scope.GlobalRegistry().InsecureRegistries - } - return insecureRegistries -} - -func (s *ContainerdService) privateRegistry() string { - if s.scope.GlobalRegistry() != nil { - return s.scope.GlobalRegistry().PrivateRegistry - } - return "" -} - -func (s *ContainerdService) auth() infrav1.RegistryAuth { - if s.scope.GlobalRegistry() != nil { - auth := s.scope.GlobalRegistry().Auth.DeepCopy() - if auth.CertsPath != "" { - ca, cert, key, err := s.lookupCertsFile(auth.CertsPath) - if err != nil { - s.scope.Info(fmt.Sprintf("Failed to lookup certs file from the specific cert path %s: %s", auth.CertsPath, err.Error())) - return *auth - } - auth.CAFile = ca - auth.CertsPath = cert - auth.KeyFile = key - } - if auth.PlainHTTP { - auth.InsecureSkipVerify = true - } - return *auth - } - return infrav1.RegistryAuth{} -} - -func (s *ContainerdService) lookupCertsFile(path string) (ca string, cert string, key string, err error) { - s.instanceScope.V(2).Info(fmt.Sprintf("Looking for TLS certificates and private keys in %s", path)) - absPath, err := filepath.Abs(path) - if err != nil { - return - } - s.instanceScope.V(2).Info(fmt.Sprintf("Looking for TLS certificates and private keys in abs path %s", absPath)) - fs, err := os.ReadDir(absPath) - if err != nil { - return ca, cert, key, err - } - - for _, f := range fs { - fullPath := filepath.Join(path, f.Name()) - if strings.HasSuffix(f.Name(), ".crt") { - s.instanceScope.V(2).Info(fmt.Sprintf(" crt: %s", fullPath)) - ca = fullPath - } - if strings.HasSuffix(f.Name(), ".cert") { - certName := f.Name() - keyName := certName[:len(certName)-5] + ".key" - s.instanceScope.V(2).Info(fmt.Sprintf(" cert: %s", fullPath)) - if !hasFile(fs, keyName) { - return ca, cert, key, errors.Errorf("missing key %s for client certificate %s. Note that CA certificates should use the extension .crt", keyName, certName) - } - cert = fullPath - } - if strings.HasSuffix(f.Name(), ".key") { - keyName := f.Name() - certName := keyName[:len(keyName)-4] + ".cert" - s.instanceScope.V(2).Info(fmt.Sprintf(" key: %s", fullPath)) - if !hasFile(fs, certName) { - return ca, cert, key, errors.Errorf("missing client certificate %s for key %s", certName, keyName) - } - key = fullPath - } - } - return ca, cert, key, nil -} - -func (s *ContainerdService) installRunc() error { - runc, err := s.getRuncService(s.sshClient, file.RuncDefaultVersion, s.instanceScope.Arch()) - if err != nil { - return err - } - - if _, err := s.sshClient.SudoCmdf("install -m 755 %s /usr/local/sbin/runc", runc.RemotePath()); err != nil { - return err - } - - _, _ = s.sshClient.SudoCmdf("rm -rf %s", runc.RemotePath()) - return nil -} - -func (s *ContainerdService) installCrictl() error { - temp, err := template.ParseFS(f, "templates/crictl.yaml") - if err != nil { - return err - } - - svc, err := s.getTemplateService( - temp, - file.Data{ - "Endpoint": s.instanceScope.ContainerManager().CRISocket, - }, - filepath.Join("/etc/", temp.Name())) - if err != nil { - return err - } - if err := svc.RenderToLocal(); err != nil { - return err - } - if err := svc.Copy(true); err != nil { - return err - } - return nil -} - -func hasFile(files []os.DirEntry, name string) bool { - for _, f := range files { - if f.Name() == name { - return true - } - } - return false -} diff --git a/pkg/service/containermanager/doc.go b/pkg/service/containermanager/doc.go deleted file mode 100644 index d8ae097f..00000000 --- a/pkg/service/containermanager/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package containermanager defines the operations on the remote instance's CRI. -package containermanager diff --git a/pkg/service/containermanager/docker.go b/pkg/service/containermanager/docker.go deleted file mode 100644 index 04ba1f77..00000000 --- a/pkg/service/containermanager/docker.go +++ /dev/null @@ -1,328 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package containermanager - -import ( - "fmt" - "path/filepath" - "strings" - "text/template" - "time" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/scope" - "github.com/kubesphere/kubekey/v3/pkg/service/operation" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/directory" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/file" - "github.com/kubesphere/kubekey/v3/pkg/service/util" -) - -// DockerService is a ContainerManager service implementation for docker. -type DockerService struct { - sshClient ssh.Interface - scope scope.KKInstanceScope - instanceScope *scope.InstanceScope - - templateFactory func(sshClient ssh.Interface, template *template.Template, data file.Data, dst string) (operation.Template, error) - dockerFactory func(sshClient ssh.Interface, version, arch string) (operation.Binary, error) - criDockerdFactory func(sshClient ssh.Interface, version, arch string) (operation.Binary, error) - crictlFactory func(sshClient ssh.Interface, version, arch string) (operation.Binary, error) -} - -// NewDockerService returns a new DockerService given the remote instance container manager client. -func NewDockerService(sshClient ssh.Interface, scope scope.KKInstanceScope, instanceScope *scope.InstanceScope) *DockerService { - return &DockerService{ - sshClient: sshClient, - scope: scope, - instanceScope: instanceScope, - } -} - -func (d *DockerService) getTemplateService(template *template.Template, data file.Data, dst string) (operation.Template, error) { - if d.templateFactory != nil { - return d.templateFactory(d.sshClient, template, data, dst) - } - return file.NewTemplate(d.sshClient, d.scope.RootFs(), template, data, dst) -} - -func (d *DockerService) getDockerService(sshClient ssh.Interface, version, arch string) (operation.Binary, error) { - if d.dockerFactory != nil { - return d.dockerFactory(sshClient, version, arch) - } - return file.NewDocker(sshClient, d.scope.RootFs(), version, arch) -} - -func (d *DockerService) getCRIDockerdService(sshClient ssh.Interface, version, arch string) (operation.Binary, error) { - if d.criDockerdFactory != nil { - return d.criDockerdFactory(sshClient, version, arch) - } - return file.NewCRIDockerd(sshClient, d.scope.RootFs(), version, arch) -} - -func (d *DockerService) getCrictlService(sshClient ssh.Interface, version, arch string) (operation.Binary, error) { - if d.crictlFactory != nil { - return d.crictlFactory(sshClient, version, arch) - } - return file.NewCrictl(sshClient, d.scope.RootFs(), version, arch) -} - -// Type returns the type docker of the container manager. -func (d *DockerService) Type() string { - return file.DockerID -} - -// Version returns the version of the container manager. -func (d *DockerService) Version() string { - return d.instanceScope.KKInstance.Spec.ContainerManager.Version -} - -// CRIDockerdVersion returns the version of the cri-dockerd. -func (d *DockerService) CRIDockerdVersion() string { - return d.instanceScope.KKInstance.Spec.ContainerManager.CRIDockerdVersion -} - -// IsExist returns true if the container manager is installed. -func (d *DockerService) IsExist() bool { - res, err := d.sshClient.SudoCmd( - "if [ -z $(which docker) ] || [ ! -e /run/docker.sock ]; " + - "then echo 'not exist'; " + - "fi") - if err != nil { - return false - } - if strings.Contains(res, "not exist") { - return false - } - return true -} - -// Get gets the binary of docker and related components and copy them to the remote instance. -func (d *DockerService) Get(timeout time.Duration) error { - docker, err := d.getDockerService(d.sshClient, d.Version(), d.instanceScope.Arch()) - if err != nil { - return err - } - criDockerd, err := d.getCRIDockerdService(d.sshClient, d.CRIDockerdVersion(), d.instanceScope.Arch()) - if err != nil { - return err - } - crictl, err := d.getCrictlService(d.sshClient, d.instanceScope.KKInstance.Spec.ContainerManager.CRICTLVersion, d.instanceScope.Arch()) - if err != nil { - return err - } - - binaries := []operation.Binary{ - docker, - criDockerd, - crictl, - } - - zone := d.scope.ComponentZone() - host := d.scope.ComponentHost() - overrideMap := make(map[string]infrav1.Override) - for _, o := range d.scope.ComponentOverrides() { - overrideMap[o.ID+o.Version+o.Arch] = o - } - - for _, b := range binaries { - override := overrideMap[b.ID()+b.Version()+b.Arch()] - if err := util.DownloadAndCopy(d.instanceScope, b, zone, host, override.Path, override.URL, override.Checksum.Value, timeout); err != nil { - return err - } - } - - // /usr/local - dir := filepath.Dir(filepath.Dir(docker.RemotePath())) - if _, err = d.sshClient.SudoCmdf("tar Cxzvf %s %s && mv %s/docker/* %s", dir, docker.RemotePath(), dir, directory.BinDir); err != nil { - return err - } - dir = filepath.Dir(filepath.Dir(criDockerd.RemotePath())) - if _, err = d.sshClient.SudoCmdf("tar Cxzvf %s %s && mv %s/cri-dockerd/* %s", dir, criDockerd.RemotePath(), dir, directory.BinDir); err != nil { - return err - } - if _, err = d.sshClient.SudoCmdf("tar Cxzvf %s %s", filepath.Dir(crictl.RemotePath()), crictl.RemotePath()); err != nil { - return err - } - return nil -} - -// Install installs the container manager and related components. -func (d *DockerService) Install() error { - if err := d.installDocker(); err != nil { - return err - } - if err := d.installCRIDockerd(); err != nil { - return err - } - if err := d.installCrictl(); err != nil { - return err - } - return nil -} - -func (d *DockerService) installDocker() error { - if err := d.generateDockerService(); err != nil { - return err - } - if err := d.generateDockerConfig(); err != nil { - return err - } - if _, err := d.sshClient.SudoCmd("systemctl daemon-reload && systemctl enable docker && systemctl start docker"); err != nil { - return err - } - return nil -} - -func (d *DockerService) installCRIDockerd() error { - if err := d.generateCRIDockerdService(); err != nil { - return err - } - if err := d.generateCRIDockerdSocket(); err != nil { - return err - } - if _, err := d.sshClient.SudoCmd("systemctl daemon-reload && systemctl enable cri-docker && systemctl enable --now cri-docker.socket && systemctl start cri-docker"); err != nil { - return err - } - return nil -} - -func (d *DockerService) installCrictl() error { - temp, err := template.ParseFS(f, "templates/crictl.yaml") - if err != nil { - return err - } - - svc, err := d.getTemplateService( - temp, - file.Data{ - "Endpoint": d.instanceScope.ContainerManager().CRISocket, - }, - filepath.Join("/etc/", temp.Name())) - if err != nil { - return err - } - if err := svc.RenderToLocal(); err != nil { - return err - } - if err := svc.Copy(true); err != nil { - return err - } - return nil -} - -func (d *DockerService) generateDockerService() error { - temp, err := template.ParseFS(f, "templates/docker.service") - if err != nil { - return err - } - svc, err := d.getTemplateService(temp, nil, filepath.Join(file.SystemdDir, temp.Name())) - if err != nil { - return err - } - if err = svc.RenderToLocal(); err != nil { - return err - } - if err = svc.Copy(true); err != nil { - return err - } - return nil -} - -func (d *DockerService) generateDockerConfig() error { - temp, err := template.ParseFS(f, "templates/daemon.json") - if err != nil { - return err - } - svc, err := d.getTemplateService( - temp, - file.Data{ - "Mirrors": d.mirrors(), - "InsecureRegistries": d.insecureRegistry(), - }, - filepath.Join("/etc/docker", temp.Name())) - if err != nil { - return err - } - if err = svc.RenderToLocal(); err != nil { - return err - } - if err = svc.Copy(true); err != nil { - return err - } - return nil -} - -func (d *DockerService) generateCRIDockerdService() error { - temp, err := template.ParseFS(f, "templates/cri-docker.service") - if err != nil { - return err - } - svc, err := d.getTemplateService(temp, nil, filepath.Join(file.SystemdDir, temp.Name())) - if err != nil { - return err - } - if err = svc.RenderToLocal(); err != nil { - return err - } - if err = svc.Copy(true); err != nil { - return err - } - return nil -} - -func (d *DockerService) generateCRIDockerdSocket() error { - temp, err := template.ParseFS(f, "templates/cri-docker.socket") - if err != nil { - return err - } - svc, err := d.getTemplateService(temp, nil, filepath.Join(file.SystemdDir, temp.Name())) - if err != nil { - return err - } - if err = svc.RenderToLocal(); err != nil { - return err - } - if err = svc.Copy(true); err != nil { - return err - } - return nil -} - -func (d *DockerService) mirrors() string { - var m string - if d.scope.GlobalRegistry() != nil { - var mirrorsArr []string - for _, mirror := range d.scope.GlobalRegistry().RegistryMirrors { - mirrorsArr = append(mirrorsArr, fmt.Sprintf("%q", mirror)) - } - m = strings.Join(mirrorsArr, ", ") - } - return m -} - -func (d *DockerService) insecureRegistry() string { - var insecureRegistries string - if d.scope.GlobalRegistry() != nil { - var registriesArr []string - for _, repo := range d.scope.GlobalRegistry().InsecureRegistries { - registriesArr = append(registriesArr, fmt.Sprintf("%q", repo)) - } - insecureRegistries = strings.Join(registriesArr, ", ") - } - return insecureRegistries -} diff --git a/pkg/service/containermanager/service.go b/pkg/service/containermanager/service.go deleted file mode 100644 index d1ef3ba2..00000000 --- a/pkg/service/containermanager/service.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package containermanager - -import ( - "embed" - "time" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/scope" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/file" -) - -//go:embed templates -var f embed.FS - -// Service holds a collection of interfaces. -// The interfaces are broken down like this to group functions together. -type Service interface { - Type() string - Version() string - IsExist() bool - Get(timeout time.Duration) error - Install() error -} - -// NewService returns a new service given the remote instance container manager client. -func NewService(sshClient ssh.Interface, scope scope.KKInstanceScope, instanceScope *scope.InstanceScope) Service { - switch instanceScope.ContainerManager().Type { - case file.ContainerdID: - return NewContainerdService(sshClient, scope, instanceScope) - case file.DockerID: - return NewDockerService(sshClient, scope, instanceScope) - default: - return NewContainerdService(sshClient, scope, instanceScope) - } -} diff --git a/pkg/service/containermanager/templates/config.toml b/pkg/service/containermanager/templates/config.toml deleted file mode 100644 index 9c54cfd4..00000000 --- a/pkg/service/containermanager/templates/config.toml +++ /dev/null @@ -1,80 +0,0 @@ -version = 2 -root = "/var/lib/containerd" -state = "/run/containerd" - -[grpc] - address = "/run/containerd/containerd.sock" - uid = 0 - gid = 0 - max_recv_message_size = 16777216 - max_send_message_size = 16777216 - -[ttrpc] - address = "" - uid = 0 - gid = 0 - -[debug] - address = "" - uid = 0 - gid = 0 - level = "" - -[metrics] - address = "" - grpc_histogram = false - -[cgroup] - path = "" - -[timeouts] - "io.containerd.timeout.shim.cleanup" = "5s" - "io.containerd.timeout.shim.load" = "5s" - "io.containerd.timeout.shim.shutdown" = "3s" - "io.containerd.timeout.task.state" = "2s" - -[plugins] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri"] - {{- if .SandBoxImage }} - sandbox_image = "{{ .SandBoxImage }}" - {{- end}} - [plugins."io.containerd.grpc.v1.cri".cni] - bin_dir = "/opt/cni/bin" - conf_dir = "/etc/cni/net.d" - max_conf_num = 1 - conf_template = "" - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - {{- if .Mirrors }} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = [{{ .Mirrors }}, "https://registry-1.docker.io"] - {{ else }} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - {{- end}} - {{- range $value := .InsecureRegistries }} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$value}}"] - endpoint = ["http://{{$value}}"] - {{- end}} - - {{- if .PrivateRegistry }} - [plugins."io.containerd.grpc.v1.cri".registry.configs] - [plugins."io.containerd.grpc.v1.cri".registry.configs.{{ .PrivateRegistry }}.auth] - username = "{{ .Auth.Username }}" - password = "{{ .Auth.Password}}" - [plugins."io.containerd.grpc.v1.cri".registry.configs.{{ .PrivateRegistry }}.tls] - {{- if .Auth.CAFile }} - ca_file = "{{ .Auth.CAFile }}" - {{- end}} - {{- if .Auth.CertFile }} - cert_file = "{{ .Auth.CertFile }}" - {{- end}} - {{- if .Auth.KeyFile }} - key_file = "{{ .Auth.KeyFile }}" - {{- end}} - insecure_skip_verify = {{ .Auth.InsecureSkipVerify }} - {{- end}} diff --git a/pkg/service/containermanager/templates/containerd.service b/pkg/service/containermanager/templates/containerd.service deleted file mode 100644 index 7f38fd71..00000000 --- a/pkg/service/containermanager/templates/containerd.service +++ /dev/null @@ -1,26 +0,0 @@ -[Unit] -Description=containerd container runtime -Documentation=https://containerd.io -After=network.target local-fs.target - -[Service] -ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/containerd - -Type=notify -Delegate=yes -KillMode=process -Restart=always -RestartSec=5 -# Having non-zero Limit*s causes performance problems due to accounting overhead -# in the kernel. We recommend using cgroups to do container-local accounting. -LimitNPROC=infinity -LimitCORE=infinity -LimitNOFILE=1048576 -# Comment TasksMax if your systemd version does not supports it. -# Only systemd 226 and above support this version. -TasksMax=infinity -OOMScoreAdjust=-999 - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/service/containermanager/templates/cri-docker.service b/pkg/service/containermanager/templates/cri-docker.service deleted file mode 100644 index 0842c128..00000000 --- a/pkg/service/containermanager/templates/cri-docker.service +++ /dev/null @@ -1,39 +0,0 @@ -[Unit] -Description=CRI Interface for Docker Application Container Engine -Documentation=https://docs.mirantis.com -After=network-online.target firewalld.service docker.service -Wants=network-online.target -Requires=cri-docker.socket - -[Service] -Type=notify -ExecStart=/usr/local/bin/cri-dockerd --container-runtime-endpoint fd:// -ExecReload=/bin/kill -s HUP $MAINPID -TimeoutSec=0 -RestartSec=2 -Restart=always - -# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. -# Both the old, and new location are accepted by systemd 229 and up, so using the old location -# to make them work for either version of systemd. -StartLimitBurst=3 - -# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. -# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make -# this option work for either version of systemd. -StartLimitInterval=60s - -# Having non-zero Limit*s causes performance problems due to accounting overhead -# in the kernel. We recommend using cgroups to do container-local accounting. -LimitNOFILE=infinity -LimitNPROC=infinity -LimitCORE=infinity - -# Comment TasksMax if your systemd version does not support it. -# Only systemd 226 and above support this option. -TasksMax=infinity -Delegate=yes -KillMode=process - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/service/containermanager/templates/cri-docker.socket b/pkg/service/containermanager/templates/cri-docker.socket deleted file mode 100644 index ddba2f5b..00000000 --- a/pkg/service/containermanager/templates/cri-docker.socket +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=CRI Docker Socket for the API -PartOf=cri-docker.service - -[Socket] -ListenStream=%t/cri-dockerd.sock -SocketMode=0660 -SocketUser=root -SocketGroup=root - -[Install] -WantedBy=sockets.target \ No newline at end of file diff --git a/pkg/service/containermanager/templates/crictl.yaml b/pkg/service/containermanager/templates/crictl.yaml deleted file mode 100644 index 236bb88f..00000000 --- a/pkg/service/containermanager/templates/crictl.yaml +++ /dev/null @@ -1,5 +0,0 @@ -runtime-endpoint: {{ .Endpoint }} -image-endpoint: {{ .Endpoint }} -timeout: 5 -debug: false -pull-image-on-create: false \ No newline at end of file diff --git a/pkg/service/containermanager/templates/daemon.json b/pkg/service/containermanager/templates/daemon.json deleted file mode 100644 index 88fb65d5..00000000 --- a/pkg/service/containermanager/templates/daemon.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "log-opts": { - "max-size": "5m", - "max-file":"3" - }, -{{- if .Mirrors }} -"registry-mirrors": [{{ .Mirrors }}], -{{- end}} -{{- if .InsecureRegistries }} -"insecure-registries": [{{ .InsecureRegistries }}], -{{- end}} -"exec-opts": ["native.cgroupdriver=systemd"] -} \ No newline at end of file diff --git a/pkg/service/containermanager/templates/docker.service b/pkg/service/containermanager/templates/docker.service deleted file mode 100644 index 6a98627e..00000000 --- a/pkg/service/containermanager/templates/docker.service +++ /dev/null @@ -1,47 +0,0 @@ -[Unit] -Description=Docker Application Container Engine -Documentation=https://docs.docker.com -# After=network-online.target firewalld.service containerd.service -# Wants=network-online.target -# Requires=docker.socket containerd.service - -[Service] -Type=notify -# the default is not to use systemd for cgroups because the delegate issues still -# exists and systemd currently does not support the cgroup feature set required -# for containers run by docker -ExecStart=/usr/local/bin/dockerd -ExecReload=/bin/kill -s HUP $MAINPID -TimeoutSec=0 -RestartSec=2 -Restart=always - -# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. -# Both the old, and new location are accepted by systemd 229 and up, so using the old location -# to make them work for either version of systemd. -StartLimitBurst=3 - -# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. -# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make -# this option work for either version of systemd. -StartLimitInterval=60s - -# Having non-zero Limit*s causes performance problems due to accounting overhead -# in the kernel. We recommend using cgroups to do container-local accounting. -LimitNOFILE=infinity -LimitNPROC=infinity -LimitCORE=infinity - -# Comment TasksMax if your systemd version does not support it. -# Only systemd 226 and above support this option. -TasksMax=infinity - -# set delegate yes so that systemd does not reset the cgroups of docker containers -Delegate=yes - -# kill only the docker process, not all processes in the cgroup -KillMode=process -OOMScoreAdjust=-500 - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/service/interface.go b/pkg/service/interface.go deleted file mode 100644 index 29411d9c..00000000 --- a/pkg/service/interface.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package service implements various services. -package service - -import ( - "time" - - "github.com/kubesphere/kubekey/v3/pkg/service/provisioning/commands" -) - -// Bootstrap is the interface for bootstrap provision. -type Bootstrap interface { - AddUsers() error - SetHostname() error - CreateDirectory() error - ResetTmpDirectory() error - ExecInitScript() error - KubeadmReset(criSocket string) error - ResetNetwork() error - RemoveFiles() error - DaemonReload() error - UninstallK3s() error -} - -// Repository is the interface for repository provision. -type Repository interface { - Check() error - Get(time time.Duration) error - MountISO() error - UmountISO() error - UpdateAndInstall() error -} - -// BinaryService is the interface for binary provision. -type BinaryService interface { - Download(timeout time.Duration) error - UpgradeDownload(timeout time.Duration) error -} - -// ContainerManager is the interface for container manager provision. -type ContainerManager interface { - Type() string - IsExist() bool - Get(timeout time.Duration) error - Install() error -} - -// Provisioning is the interface for bootstrap generate by CABPK provision. -type Provisioning interface { - RawBootstrapDataToProvisioningCommands(config []byte) ([]commands.Cmd, error) -} diff --git a/pkg/service/operation/directory/consts.go b/pkg/service/operation/directory/consts.go deleted file mode 100644 index 0a41d1c6..00000000 --- a/pkg/service/operation/directory/consts.go +++ /dev/null @@ -1,34 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package directory - -const ( - // TmpDir represents the tmp directory of the remote instance - TmpDir = "/tmp/kubekey" - // BinDir represents the bin directory of the remote instance - BinDir = "/usr/local/bin" - // KubeConfigDir represents the normal kubernetes data directory of the remote instance - KubeConfigDir = "/etc/kubernetes" - // KubeCertDir represents the normal kubernetes cert directory of the remote instance - KubeCertDir = "/etc/kubernetes/pki" - // KubeManifestDir represents the normal kubernetes manifest directory of the remote instance - KubeManifestDir = "/etc/kubernetes/manifests" - // KubeScriptDir represents the kubernetes manage tools scripts directory of the remote instance - KubeScriptDir = "/usr/local/bin/kube-scripts" - // KubeletFlexvolumesPluginsDir represents the kubernetes kubelet plugin volume directory of the remote instance - KubeletFlexvolumesPluginsDir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec" -) diff --git a/pkg/service/operation/directory/directory.go b/pkg/service/operation/directory/directory.go deleted file mode 100644 index 6e08d9ce..00000000 --- a/pkg/service/operation/directory/directory.go +++ /dev/null @@ -1,52 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package directory - -import ( - "fmt" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/pkg/util/filesystem" -) - -// Make wraps the Linux command "mkdir -p -m ". -func (s *Service) Make() error { - _, err := s.sshClient.SudoCmdf("mkdir -p -m %o %s", filesystem.ToChmodPerm(s.mode), s.path) - if err != nil { - return errors.Wrapf(err, "failed to mkdir -p -m %o %s", filesystem.ToChmodPerm(s.mode), s.path) - } - return nil -} - -// Chown wraps the linux command "chown -R ". -func (s *Service) Chown(user string) error { - _, err := s.sshClient.SudoCmd(fmt.Sprintf("chown %s -R %s", user, s.path)) - if err != nil { - return errors.Wrapf(err, "failed to chown %s -R %s", user, s.path) - } - return nil -} - -// Remove wraps the linux command "rm -rf ". -func (s *Service) Remove() error { - _, err := s.sshClient.SudoCmd(fmt.Sprintf("rm -rf %s", s.path)) - if err != nil { - return errors.Wrapf(err, "failed to rm -rf %s", s.path) - } - return nil -} diff --git a/pkg/service/operation/directory/doc.go b/pkg/service/operation/directory/doc.go deleted file mode 100644 index 5f467f45..00000000 --- a/pkg/service/operation/directory/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package directory defines the operations on remote instance directory. -package directory diff --git a/pkg/service/operation/directory/service.go b/pkg/service/operation/directory/service.go deleted file mode 100644 index 0e7e76c1..00000000 --- a/pkg/service/operation/directory/service.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package directory - -import ( - "os" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/util/filesystem" -) - -// Service holds a collection of interfaces. -// The interfaces are broken down like this to group functions together. -type Service struct { - sshClient ssh.Interface - path string - mode os.FileMode -} - -// NewService returns a new service given the remote instance directory client. -func NewService(sshClient ssh.Interface, path string, mode os.FileMode) *Service { - return &Service{ - sshClient: sshClient, - path: path, - mode: checkFileMode(mode), - } -} - -func checkFileMode(mode os.FileMode) os.FileMode { - if mode.Perm() == 0 { - mode = os.FileMode(filesystem.FileMode0664) - } - - if mode.Type() != os.ModeDir { - mode = os.ModeDir | mode - } - - return mode -} diff --git a/pkg/service/operation/directory/service_test.go b/pkg/service/operation/directory/service_test.go deleted file mode 100644 index c0c6dd52..00000000 --- a/pkg/service/operation/directory/service_test.go +++ /dev/null @@ -1,49 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package directory - -import ( - "os" - "testing" -) - -func Test_checkFileMode(t *testing.T) { - tests := []struct { - mode os.FileMode - want os.FileMode - }{ - { - 0, - os.ModeDir | os.FileMode(0664), - }, - { - os.FileMode(0664), - os.ModeDir | os.FileMode(0664), - }, - { - os.FileMode(0777), - os.ModeDir | os.FileMode(0777), - }, - } - for _, tt := range tests { - t.Run("", func(t *testing.T) { - if got := checkFileMode(tt.mode); got != tt.want { - t.Errorf("checkFileMode() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/service/operation/file/binary.go b/pkg/service/operation/file/binary.go deleted file mode 100644 index 1ae423c4..00000000 --- a/pkg/service/operation/file/binary.go +++ /dev/null @@ -1,259 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package file - -import ( - "crypto/sha256" - "fmt" - "io" - "net/url" - "os" - "path" - "reflect" - "strings" - "time" - - "github.com/hashicorp/go-getter" - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/pkg/service/operation/file/checksum" -) - -// Default -const ( - ZONE = "cn" - DefaultDownloadHost = "https://github.com" - DefaultDownloadHostGoogle = "https://storage.googleapis.com" - DefaultDownloadHostQingStor = "https://kubernetes-release.pek3b.qingstor.com" -) - -// BinaryParams represents the parameters of a Binary. -type BinaryParams struct { - File *File - ID string - Version string - Arch string - URL *url.URL - ChecksumList []checksum.Interface -} - -// NewBinary returns a new Binary. -func NewBinary(params BinaryParams) *Binary { - b := &Binary{ - file: params.File, - id: params.ID, - version: params.Version, - arch: params.Arch, - url: params.URL, - checksumList: checksum.NewChecksums(checksum.NewInternalChecksum(params.ID, params.Version, params.Arch)), - } - if len(params.ChecksumList) != 0 { - b.checksumList.Append(params.ChecksumList...) - } - return b -} - -// Binary is a binary implementation of Binary interface. -type Binary struct { - file *File - id string - version string - arch string - url *url.URL - checksumList checksum.List -} - -// Name returns the name of the Binary file. -func (b *Binary) Name() string { - return b.file.Name() -} - -// Type returns the type of the Binary file. -func (b *Binary) Type() Type { - return b.file.Type() -} - -// LocalPath returns the local path of the Binary file. -func (b *Binary) LocalPath() string { - return b.file.LocalPath() -} - -// RemotePath returns the remote path of the Binary file. -func (b *Binary) RemotePath() string { - return b.file.RemotePath() -} - -// LocalExist returns true if the Binary file is existed in the local path. -func (b *Binary) LocalExist() bool { - return b.file.LocalExist() -} - -// RemoteExist returns true if the Binary file is existed (and the SHA256 check passes) in the remote path. -func (b *Binary) RemoteExist() bool { - if !b.file.RemoteExist() { - return false - } - - cmd := fmt.Sprintf("sha256sum %s | cut -d\" \" -f1", b.file.RemotePath()) - remoteSHA256, err := b.file.sshClient.SudoCmd(cmd) - if err != nil { - return false - } - - if err := b.checksumList.Get(); err != nil { - return false - } - - if remoteSHA256 != b.checksumList.Value() { - return false - } - return true -} - -// Copy copies the Binary file from the local path to the remote path. -func (b *Binary) Copy(override bool) error { - return b.file.Copy(override) -} - -// Fetch copies the Binary file from the remote path to the local path. -func (b *Binary) Fetch(override bool) error { - return b.file.Fetch(override) -} - -// Chmod changes the mode of the Binary file. -func (b *Binary) Chmod(option string) error { - return b.file.Chmod(option) -} - -// ID returns the id of the binary. -func (b *Binary) ID() string { - return b.id -} - -// Arch returns the arch of the binary. -func (b *Binary) Arch() string { - return b.arch -} - -// Version returns the version of the binary. -func (b *Binary) Version() string { - return b.version -} - -// URL returns the download url of the binary. -func (b *Binary) URL() *url.URL { - return b.url -} - -// SetURL sets the download url of the binary. -func (b *Binary) SetURL(urlStr string) { - if urlStr == "" { - return - } - u, err := url.Parse(urlStr) - if err != nil { - return - } - b.url = u -} - -// SetHost sets the host to download the binaries. -func (b *Binary) SetHost(host string) { - if host == "" { - return - } - u, err := url.Parse(host) - if err != nil { - return - } - u.Path = b.url.Path - b.url = u -} - -// SetPath sets the URL path of the binary. -func (b *Binary) SetPath(pathStr string) { - if pathStr == "" { - return - } - ref, err := url.Parse(pathStr) - if err != nil { - return - } - b.url = b.url.ResolveReference(ref) -} - -// SetZone sets the zone of the binary. -func (b *Binary) SetZone(zone string) { - if strings.EqualFold(zone, ZONE) { - b.SetHost(DefaultDownloadHostQingStor) - } -} - -// AppendChecksum appends a checksum to the checksum list. -func (b *Binary) AppendChecksum(c checksum.Interface) { - if reflect.ValueOf(c).IsNil() { - return - } - b.checksumList.Append(c) -} - -// Get downloads the binary from remote. -func (b *Binary) Get(timeout time.Duration) error { - client := &getter.HttpGetter{ - ReadTimeout: timeout, - } - - if err := client.GetFile(b.LocalPath(), b.URL()); err != nil { - return errors.Wrapf(err, "failed to http get file: %s", b.URL()) - } - - return nil -} - -// SHA256 calculates the SHA256 of the binary. -func (b *Binary) SHA256() (string, error) { - f, err := os.Open(b.LocalPath()) - if err != nil { - return "", err - } - defer f.Close() - - data, err := io.ReadAll(f) - if err != nil { - return "", err - } - return fmt.Sprintf("%x", sha256.Sum256(data)), nil -} - -// CompareChecksum compares the checksum of the binary. -func (b *Binary) CompareChecksum() error { - if err := b.checksumList.Get(); err != nil { - return errors.Wrapf(err, "%s get checksum failed", b.Name()) - } - - sum := b.file.rootFs.Fs().SHA256Sum(b.LocalPath()) - if sum != b.checksumList.Value() { - return errors.Errorf("SHA256 no match. file: %s sha256: %s not equal checksum: %s", b.Name(), sum, b.checksumList.Value()) - } - return nil -} - -func parseURL(host, pathStr string) *url.URL { - u, _ := url.Parse(host) - u.Path = path.Join(u.Path, pathStr) - return u -} diff --git a/pkg/service/operation/file/checksum/checksums.go b/pkg/service/operation/file/checksum/checksums.go deleted file mode 100644 index af4774f0..00000000 --- a/pkg/service/operation/file/checksum/checksums.go +++ /dev/null @@ -1,60 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package checksum - -import ( - kerrors "k8s.io/apimachinery/pkg/util/errors" -) - -// Checksums is a list of checksums. -type Checksums struct { - checksums []Interface - value string -} - -// NewChecksums returns a new Checksums. -func NewChecksums(checksums ...Interface) *Checksums { - return &Checksums{checksums: checksums} -} - -// Get gets the checksums. It will iterate through the Get() of the []Interface array and store the first fetched value. -func (c *Checksums) Get() error { - if c.value != "" { - return nil - } - - var errs []error - for _, v := range c.checksums { - if err := v.Get(); err != nil { - errs = append(errs, err) - continue - } - c.value = v.Value() - return nil - } - return kerrors.NewAggregate(errs) -} - -// Value returns the checksums. -func (c *Checksums) Value() string { - return c.value -} - -// Append appends checksums. -func (c *Checksums) Append(checksums ...Interface) { - c.checksums = append(c.checksums, checksums...) -} diff --git a/pkg/service/operation/file/checksum/doc.go b/pkg/service/operation/file/checksum/doc.go deleted file mode 100644 index 3bdb6d7b..00000000 --- a/pkg/service/operation/file/checksum/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package checksum implements methods for obtaining and verifying the checksum of binary files. -package checksum diff --git a/pkg/service/operation/file/checksum/http_checksum.go b/pkg/service/operation/file/checksum/http_checksum.go deleted file mode 100644 index 71d4f9d3..00000000 --- a/pkg/service/operation/file/checksum/http_checksum.go +++ /dev/null @@ -1,156 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package checksum - -import ( - "bufio" - "fmt" - "io" - "net/url" - "os" - "path/filepath" - "strings" - "time" - - "github.com/hashicorp/go-getter" - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/pkg/rootfs" -) - -// HTTPChecksum is a checksum that is downloaded from a URL. -type HTTPChecksum struct { - fs rootfs.Interface - url *url.URL - FileName string - value string -} - -// NewHTTPChecksum returns a new HTTPChecksum. -func NewHTTPChecksum(url *url.URL, isoName string, fs rootfs.Interface) *HTTPChecksum { - return &HTTPChecksum{ - url: url, - FileName: isoName, - fs: fs, - } -} - -// SetHost sets the host of the URL. -func (h *HTTPChecksum) SetHost(host string) { - if host == "" { - return - } - u, err := url.Parse(host) - if err != nil { - return - } - u.Path = h.url.Path - h.url = u -} - -// SetPath sets the URL path of the binary. -func (h *HTTPChecksum) SetPath(pathStr string) { - if pathStr == "" { - return - } - ref, err := url.Parse(pathStr) - if err != nil { - return - } - - h.url = h.url.ResolveReference(ref) -} - -// Get downloads the checksum file and parses it. -func (h *HTTPChecksum) Get() error { - tempfile, err := h.fs.Fs().MkLocalTmpFile(h.fs.ClusterRootFsDir(), filepath.Base(h.url.Path)) - if err != nil { - return err - } - defer func() { - _ = h.fs.Fs().RemoveAll(tempfile) - }() - - client := &getter.HttpGetter{ - ReadTimeout: 15 * time.Second, - } - - if err := client.GetFile(tempfile, h.url); err != nil { - return errors.Wrapf(err, "failed to http get file: %s", h.url) - } - - f, err := os.Open(filepath.Clean(tempfile)) - if err != nil { - return fmt.Errorf("error opening downloaded file: %s", err) - } - defer f.Close() - rd := bufio.NewReader(f) - for { - line, err := rd.ReadString('\n') - if err != nil { - if err != io.EOF { - return fmt.Errorf("error reading checksum file: %s", err) - } - if line == "" { - break - } - // parse the line, if we hit EOF, but the line is not empty - } - checksum, filename, err := parseChecksumLine(line) - if err != nil || checksum == "" { - continue - } - if filename == h.FileName { - h.value = checksum - return nil - } - } - return fmt.Errorf("no checksum found in: %s", h.url.String()) -} - -// Value returns the checksum value. -func (h *HTTPChecksum) Value() string { - return h.value -} - -// parseChecksumLine takes a line from a checksum file and returns the checksum and filename. -func parseChecksumLine(line string) (string, string, error) { - parts := strings.Fields(line) - - switch len(parts) { - case 4: - // BSD-style checksum: - // MD5 (file1) = - // MD5 (file2) = - if len(parts[1]) <= 2 || - parts[1][0] != '(' || parts[1][len(parts[1])-1] != ')' { - return "", "", fmt.Errorf( - "unexpected BSD-style-checksum filename format: %s", line) - } - filename := parts[1][1 : len(parts[1])-1] - return parts[3], filename, nil - case 2: - // GNU-style: - // file1 - // *file2 - return parts[0], parts[1], nil - case 0: - return "", "", nil - default: - return parts[0], "", nil - } -} diff --git a/pkg/service/operation/file/checksum/interface.go b/pkg/service/operation/file/checksum/interface.go deleted file mode 100644 index aba6dd7e..00000000 --- a/pkg/service/operation/file/checksum/interface.go +++ /dev/null @@ -1,29 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package checksum - -// Interface is the interface for checksum. -type Interface interface { - Get() error - Value() string -} - -// List is the interface of a list of checksums. -type List interface { - Interface - Append(checksum ...Interface) -} diff --git a/pkg/service/operation/file/checksum/internal_checksum.go b/pkg/service/operation/file/checksum/internal_checksum.go deleted file mode 100644 index 358deaeb..00000000 --- a/pkg/service/operation/file/checksum/internal_checksum.go +++ /dev/null @@ -1,64 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package checksum - -import ( - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/version" -) - -var ( - // FileSha256 is a hash table the storage the checksum of the binary files. It is parsed from 'version/components.json'. - FileSha256 = map[string]map[string]map[string]string{} -) - -func init() { - FileSha256, _ = version.ParseFilesSha256(version.Components) -} - -// InternalChecksum is the internal checksum implementation. -type InternalChecksum struct { - ID string - Version string - Arch string - value string -} - -// NewInternalChecksum returns a new internal checksum implementation given the binary information. -func NewInternalChecksum(id, version, arch string) *InternalChecksum { - return &InternalChecksum{ - ID: id, - Version: version, - Arch: arch, - } -} - -// Get gets the internal checksum. -func (i *InternalChecksum) Get() error { - value, ok := FileSha256[i.ID][i.Arch][i.Version] - if !ok { - return errors.New("unsupported version") - } - i.value = value - return nil -} - -// Value returns the internal checksum value. -func (i *InternalChecksum) Value() string { - return i.value -} diff --git a/pkg/service/operation/file/checksum/string_checksum.go b/pkg/service/operation/file/checksum/string_checksum.go deleted file mode 100644 index b0bd23e3..00000000 --- a/pkg/service/operation/file/checksum/string_checksum.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package checksum - -// StringChecksum is a simple checksum implementation for given strings. -type StringChecksum struct { - value string -} - -// NewStringChecksum returns a new StringChecksum for the given string. -func NewStringChecksum(value string) *StringChecksum { - if value == "" { - return nil - } - return &StringChecksum{value: value} -} - -// Get the StringChecksum Get() method will do nothing. -func (s *StringChecksum) Get() error { - return nil -} - -// Value returns the value of the StringChecksum. -func (s *StringChecksum) Value() string { - return s.value -} diff --git a/pkg/service/operation/file/consts.go b/pkg/service/operation/file/consts.go deleted file mode 100644 index 0cc510a2..00000000 --- a/pkg/service/operation/file/consts.go +++ /dev/null @@ -1,25 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package file - -// Common dir -const ( - BinDir = "/usr/local/bin" - OptCniBinDir = "/opt/cni/bin" - SystemdDir = "/etc/systemd/system" - MntDir = "/mnt/kubekey" -) diff --git a/pkg/service/operation/file/containerd.go b/pkg/service/operation/file/containerd.go deleted file mode 100644 index 78264fe1..00000000 --- a/pkg/service/operation/file/containerd.go +++ /dev/null @@ -1,64 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package file - -import ( - "fmt" - "path/filepath" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/rootfs" -) - -// Containerd info -const ( - ContainerdName = "containerd-%s-linux-%s.tar.gz" - ContainerdID = "containerd" - ContainerdURLPathTmpl = "/containerd/containerd/releases/download/v%s/containerd-%s-linux-%s.tar.gz" -) - -// Containerd is a Binary for containerd. -type Containerd struct { - *Binary -} - -// NewContainerd returns a new Containerd. -func NewContainerd(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch string) (*Containerd, error) { - fileName := fmt.Sprintf(ContainerdName, version, arch) - file, err := NewFile(Params{ - SSHClient: sshClient, - RootFs: rootFs, - Type: FileBinary, - Name: fileName, - LocalFullPath: filepath.Join(rootFs.ClusterRootFsDir(), ContainerdID, version, arch, fileName), - RemoteFullPath: filepath.Join(BinDir, fileName), - }) - if err != nil { - return nil, err - } - - u := parseURL(DefaultDownloadHost, fmt.Sprintf(ContainerdURLPathTmpl, version, version, arch)) - binary := NewBinary(BinaryParams{ - File: file, - ID: ContainerdID, - Version: version, - Arch: arch, - URL: u, - }) - - return &Containerd{binary}, nil -} diff --git a/pkg/service/operation/file/crictl.go b/pkg/service/operation/file/crictl.go deleted file mode 100644 index a0084a84..00000000 --- a/pkg/service/operation/file/crictl.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package file - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/rootfs" -) - -// Crictl info -const ( - CrictlName = "crictl-%s-linux-%s.tar.gz" - CrictlID = "crictl" - CrictlURLPathTmpl = "/kubernetes-sigs/cri-tools/releases/download/%s/crictl-%s-linux-%s.tar.gz" - CrictlURLPathTmplCN = "/cri-tools/releases/download/%s/crictl-%s-linux-%s.tar.gz" -) - -// Crictl is a Binary for crictl. -type Crictl struct { - *Binary -} - -// NewCrictl returns a new Crictl. -func NewCrictl(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch string) (*Crictl, error) { - fileName := fmt.Sprintf(CrictlName, version, arch) - file, err := NewFile(Params{ - SSHClient: sshClient, - RootFs: rootFs, - Type: FileBinary, - Name: fileName, - LocalFullPath: filepath.Join(rootFs.ClusterRootFsDir(), CrictlID, version, arch, fileName), - RemoteFullPath: filepath.Join(BinDir, fileName), - }) - if err != nil { - return nil, err - } - - u := parseURL(DefaultDownloadHost, fmt.Sprintf(CrictlURLPathTmpl, version, version, arch)) - binary := NewBinary(BinaryParams{ - File: file, - ID: CrictlID, - Version: version, - Arch: arch, - URL: u, - }) - - return &Crictl{binary}, nil -} - -// SetZone override Binary's SetZone method. -func (c *Crictl) SetZone(zone string) { - if strings.EqualFold(zone, ZONE) { - c.SetHost(DefaultDownloadHostQingStor) - c.SetPath(fmt.Sprintf(CrictlURLPathTmplCN, c.version, c.version, c.arch)) - } -} diff --git a/pkg/service/operation/file/doc.go b/pkg/service/operation/file/doc.go deleted file mode 100644 index 1203c393..00000000 --- a/pkg/service/operation/file/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package file defines the operations on the binaries. -package file diff --git a/pkg/service/operation/file/docker.go b/pkg/service/operation/file/docker.go deleted file mode 100644 index 1dd42243..00000000 --- a/pkg/service/operation/file/docker.go +++ /dev/null @@ -1,124 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package file - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/rootfs" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/directory" - "github.com/kubesphere/kubekey/v3/pkg/util" -) - -// Docker info -const ( - DockerName = "docker-%s.tgz" - DockerID = "docker" - DockerURL = "https://download.docker.com" - DockerURLPathTmpl = "/linux/static/stable/%s/docker-%s.tgz" - DockerURLCN = "https://mirrors.aliyun.com" - DockerURLPathTmplCN = "/docker-ce/linux/static/stable/%s/docker-%s.tgz" - DockerDefaultVersion = "20.10.8" -) - -// CRI-Dockerd Info -const ( - CRIDockerdName = "cri-dockerd-%s.%s.tgz" - CRIDockerdID = "cri-dockerd" - CRIDockerdURL = "https://github.com/Mirantis/cri-dockerd/releases/download" - CRIDockerdURLPathTmpl = "/v%s/cri-dockerd-%s.%s.tgz" - DefaultCRIDockerdVersion = "0.2.6" -) - -// Docker is a Binary for docker. -type Docker struct { - *Binary -} - -// NewDocker returns a new Docker. -func NewDocker(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch string) (*Docker, error) { - fileName := fmt.Sprintf(DockerName, version) - file, err := NewFile(Params{ - SSHClient: sshClient, - RootFs: rootFs, - Type: FileBinary, - Name: fileName, - LocalFullPath: filepath.Join(rootFs.ClusterRootFsDir(), DockerID, version, arch, fileName), - RemoteFullPath: filepath.Join(directory.BinDir, fileName), - }) - if err != nil { - return nil, err - } - - u := parseURL(DockerURL, fmt.Sprintf(DockerURLPathTmpl, util.ArchAlias(arch), version)) - binary := NewBinary(BinaryParams{ - File: file, - ID: DockerID, - Version: version, - Arch: arch, - URL: u, - }) - - return &Docker{binary}, nil -} - -// SetZone override Binary's SetZone method. -func (d *Docker) SetZone(zone string) { - if strings.EqualFold(zone, ZONE) { - d.SetHost(DockerURLCN) - d.SetPath(fmt.Sprintf(DockerURLPathTmplCN, util.ArchAlias(d.arch), d.version)) - } -} - -// CRIDockerd is a Binary for cri-dockerd. -type CRIDockerd struct { - *Binary -} - -// NewCRIDockerd returns a new CRIDockerd. -func NewCRIDockerd(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch string) (*CRIDockerd, error) { - fileName := fmt.Sprintf(CRIDockerdName, version, arch) - file, err := NewFile(Params{ - SSHClient: sshClient, - RootFs: rootFs, - Type: FileBinary, - Name: fileName, - LocalFullPath: filepath.Join(rootFs.ClusterRootFsDir(), CRIDockerdID, version, arch, fileName), - RemoteFullPath: filepath.Join(directory.BinDir, fileName), - }) - if err != nil { - return nil, err - } - - u := parseURL(CRIDockerdURL, fmt.Sprintf(CRIDockerdURLPathTmpl, version, version, arch)) - binary := NewBinary(BinaryParams{ - File: file, - ID: CRIDockerdID, - Version: version, - Arch: arch, - URL: u, - }) - return &CRIDockerd{binary}, nil -} - -// SetZone override Binary's SetZone method. -func (d *CRIDockerd) SetZone(zone string) { - // TODO set zone -} diff --git a/pkg/service/operation/file/file.go b/pkg/service/operation/file/file.go deleted file mode 100644 index 39abbd0f..00000000 --- a/pkg/service/operation/file/file.go +++ /dev/null @@ -1,175 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package file - -import ( - "os" - "reflect" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/rootfs" -) - -// Type represents the type of file. -type Type string - -var ( - // FileBinary represents a binary file. - FileBinary = Type("fileBinary") - // FileText represents a text file. - FileText = Type("fileText") - // FileTemplate represents a template file. - FileTemplate = Type("fileTemplate") -) - -// Params represents the parameters of a file. -type Params struct { - SSHClient ssh.Interface - Name string - Type Type - LocalFullPath string - RemoteFullPath string - RootFs rootfs.Interface -} - -// NewFile returns a new File object given a FileParams. -func NewFile(params Params) (*File, error) { - if reflect.ValueOf(params.SSHClient).IsNil() { - return nil, errors.New("ssh client is required when creating a File") - } - if reflect.ValueOf(params.RootFs).IsNil() { - return nil, errors.New("rootfs is required when creating a File") - } - if params.Name == "" { - return nil, errors.New("name is required when creating a File") - } - if params.Type == "" { - return nil, errors.New("file type is required when creating a File") - } - return &File{ - sshClient: params.SSHClient, - rootFs: params.RootFs, - name: params.Name, - fileType: params.Type, - localFullPath: params.LocalFullPath, - remoteFullPath: params.RemoteFullPath, - }, nil -} - -// File is an implementation of the File interface. -type File struct { - sshClient ssh.Interface - name string - fileType Type - localFullPath string - remoteFullPath string - rootFs rootfs.Interface -} - -// Name returns the name of the file. -func (s *File) Name() string { - return s.name -} - -// Type returns the type of the file. -func (s *File) Type() Type { - return s.fileType -} - -// SetLocalPath sets the local path of the file. -func (s *File) SetLocalPath(path string) { - s.localFullPath = path -} - -// SetRemotePath sets the remote path of the file. -func (s *File) SetRemotePath(path string) { - s.remoteFullPath = path -} - -// LocalPath returns the local path of the file. -func (s *File) LocalPath() string { - return s.localFullPath -} - -// RemotePath returns the remote path of the file. -func (s *File) RemotePath() string { - return s.remoteFullPath -} - -// LocalExist returns true if the file exists in the local path. -func (s *File) LocalExist() bool { - _, err := os.Stat(s.LocalPath()) - if err != nil { - if os.IsExist(err) { - return true - } - if os.IsNotExist(err) { - return false - } - return false - } - return true -} - -// RemoteExist returns true if the file exists in the remote path. -func (s *File) RemoteExist() bool { - ok, err := s.sshClient.RemoteFileExist(s.RemotePath()) - if err != nil { - return false - } - return ok -} - -// Copy copies the file from the local path to the remote path. -func (s *File) Copy(override bool) error { - if !s.LocalExist() { - return errors.Errorf("file %s is not exist in the local path %s", s.Name(), s.LocalPath()) - } - - if !override { - if s.RemoteExist() { - return nil - } - } - return s.sshClient.Copy(s.LocalPath(), s.RemotePath()) -} - -// Fetch copies the file from the remote path to the local path. -func (s *File) Fetch(override bool) error { - if !s.RemoteExist() { - return errors.Errorf("remote file %s is not exist in the remote path %s", s.Name(), s.RemotePath()) - } - - if !override { - if s.LocalExist() { - return nil - } - } - return s.sshClient.Fetch(s.LocalPath(), s.RemotePath()) -} - -// Chmod changes the mode of the file. -func (s *File) Chmod(option string) error { - if !s.RemoteExist() { - return errors.Errorf("remote file %s is not exist in the remote path %s", s.Name(), s.RemotePath()) - } - - _, err := s.sshClient.SudoCmdf("chmod %s %s", option, s.remoteFullPath) - return err -} diff --git a/pkg/service/operation/file/k3s.go b/pkg/service/operation/file/k3s.go deleted file mode 100644 index 0b305f45..00000000 --- a/pkg/service/operation/file/k3s.go +++ /dev/null @@ -1,79 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package file - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/rootfs" -) - -// K3s info -const ( - K3sName = "k3s" - K3sID = "k3s" - K3sURLPathTmpl = "/k3s-io/k3s/releases/download/%s+k3s1/k3s%s" - K3sURLPathTmplCN = "/k3s/releases/download/%s+k3s1/linux/%s/k3s" -) - -// K3s is a Binary for k3s. -type K3s struct { - *Binary -} - -// NewK3s returns a new K3s. -func NewK3s(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch string) (*K3s, error) { - fileName := K3sName - file, err := NewFile(Params{ - SSHClient: sshClient, - RootFs: rootFs, - Type: FileBinary, - Name: fileName, - LocalFullPath: filepath.Join(rootFs.ClusterRootFsDir(), K3sID, version, arch, fileName), - RemoteFullPath: filepath.Join(BinDir, fileName), - }) - if err != nil { - return nil, err - } - - var urlArchPath string - if arch != "amd64" { - urlArchPath = "-" + arch - } - - u := parseURL(DefaultDownloadHost, fmt.Sprintf(K3sURLPathTmpl, version, urlArchPath)) - binary := NewBinary(BinaryParams{ - File: file, - ID: K3sID, - Version: version, - Arch: arch, - URL: u, - }) - - return &K3s{binary}, nil -} - -// SetZone override Binary's SetZone method. -func (k *K3s) SetZone(zone string) { - if strings.EqualFold(zone, ZONE) { - k.SetHost(DefaultDownloadHostQingStor) - k.SetPath(fmt.Sprintf(K3sURLPathTmplCN, k.version, k.arch)) - } -} diff --git a/pkg/service/operation/file/kubeadm.go b/pkg/service/operation/file/kubeadm.go deleted file mode 100644 index 06dfb01c..00000000 --- a/pkg/service/operation/file/kubeadm.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package file - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/rootfs" -) - -// Kubeadm info -const ( - KubeadmName = "kubeadm" - KubeadmID = "kubeadm" - KubeadmURLPathTmpl = "/kubernetes-release/release/%s/bin/linux/%s/kubeadm" - KubeadmURLPathTmplCN = "/release/%s/bin/linux/%s/kubeadm" -) - -// Kubeadm is a Binary for kubeadm. -type Kubeadm struct { - *Binary -} - -// NewKubeadm returns a new Kubeadm. -func NewKubeadm(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch string) (*Kubeadm, error) { - fileName := KubeadmName - file, err := NewFile(Params{ - SSHClient: sshClient, - RootFs: rootFs, - Type: FileBinary, - Name: fileName, - LocalFullPath: filepath.Join(rootFs.ClusterRootFsDir(), KubeadmID, version, arch, fileName), - RemoteFullPath: filepath.Join(BinDir, fileName), - }) - if err != nil { - return nil, err - } - - u := parseURL(DefaultDownloadHostGoogle, fmt.Sprintf(KubeadmURLPathTmpl, version, arch)) - binary := NewBinary(BinaryParams{ - File: file, - ID: KubeadmID, - Version: version, - Arch: arch, - URL: u, - }) - - return &Kubeadm{binary}, nil -} - -// SetZone override Binary's SetZone method. -func (k *Kubeadm) SetZone(zone string) { - if strings.EqualFold(zone, ZONE) { - k.SetHost(DefaultDownloadHostQingStor) - k.SetPath(fmt.Sprintf(KubeadmURLPathTmplCN, k.version, k.arch)) - } -} diff --git a/pkg/service/operation/file/kubecni.go b/pkg/service/operation/file/kubecni.go deleted file mode 100644 index 8b6616b7..00000000 --- a/pkg/service/operation/file/kubecni.go +++ /dev/null @@ -1,76 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package file - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/rootfs" -) - -// Kubecni info -const ( - KubecniName = "cni-plugins-linux-%s-%s.tgz" - KubecniID = "kubecni" - KubecniURLPathTmpl = "/containernetworking/plugins/releases/download/%s/cni-plugins-linux-%s-%s.tgz" - KubecniURLCN = "https://containernetworking.pek3b.qingstor.com" - KubecniURLPathTmplCN = "/plugins/releases/download/%s/cni-plugins-linux-%s-%s.tgz" - KubecniDefaultVersion = "v1.2.0" -) - -// Kubecni is a Binary for kubecni. -type Kubecni struct { - *Binary -} - -// NewKubecni returns a new Kubecni. -func NewKubecni(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch string) (*Kubecni, error) { - fileName := fmt.Sprintf(KubecniName, arch, version) - file, err := NewFile(Params{ - SSHClient: sshClient, - RootFs: rootFs, - Type: FileBinary, - Name: fileName, - LocalFullPath: filepath.Join(rootFs.ClusterRootFsDir(), KubecniID, version, arch, fileName), - RemoteFullPath: filepath.Join(OptCniBinDir, fileName), - }) - if err != nil { - return nil, err - } - - u := parseURL(DefaultDownloadHost, fmt.Sprintf(KubecniURLPathTmpl, version, arch, version)) - binary := NewBinary(BinaryParams{ - File: file, - ID: KubecniID, - Version: version, - Arch: arch, - URL: u, - }) - - return &Kubecni{binary}, nil -} - -// SetZone override Binary's SetZone method. -func (k *Kubecni) SetZone(zone string) { - if strings.EqualFold(zone, ZONE) { - k.SetHost(KubecniURLCN) - k.SetPath(fmt.Sprintf(KubecniURLPathTmplCN, k.version, k.arch, k.version)) - } -} diff --git a/pkg/service/operation/file/kubectl.go b/pkg/service/operation/file/kubectl.go deleted file mode 100644 index 6590ef0c..00000000 --- a/pkg/service/operation/file/kubectl.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package file - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/rootfs" -) - -// Kubectl info -const ( - KubectlName = "kubectl" - KubectlID = "kubectl" - KubectlURLPathTmpl = "/kubernetes-release/release/%s/bin/linux/%s/kubectl" - KubectlURLPathTmplCN = "/release/%s/bin/linux/%s/kubectl" -) - -// Kubectl is a Binary for kubectl. -type Kubectl struct { - *Binary -} - -// NewKubectl returns a new Kubectl. -func NewKubectl(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch string) (*Kubectl, error) { - fileName := KubectlName - file, err := NewFile(Params{ - SSHClient: sshClient, - RootFs: rootFs, - Type: FileBinary, - Name: fileName, - LocalFullPath: filepath.Join(rootFs.ClusterRootFsDir(), KubectlID, version, arch, fileName), - RemoteFullPath: filepath.Join(BinDir, fileName), - }) - if err != nil { - return nil, err - } - - u := parseURL(DefaultDownloadHostGoogle, fmt.Sprintf(KubectlURLPathTmpl, version, arch)) - binary := NewBinary(BinaryParams{ - File: file, - ID: KubectlID, - Version: version, - Arch: arch, - URL: u, - }) - - return &Kubectl{binary}, nil -} - -// SetZone override Binary's SetZone method. -func (k *Kubectl) SetZone(zone string) { - if strings.EqualFold(zone, ZONE) { - k.SetHost(DefaultDownloadHostQingStor) - k.SetPath(fmt.Sprintf(KubectlURLPathTmplCN, k.version, k.arch)) - } -} diff --git a/pkg/service/operation/file/kubelet.go b/pkg/service/operation/file/kubelet.go deleted file mode 100644 index 3847dc11..00000000 --- a/pkg/service/operation/file/kubelet.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package file - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/rootfs" -) - -// kubelet info -const ( - KubeletName = "kubelet" - KubeletID = "kubelet" - KubeletURLPathTmpl = "/kubernetes-release/release/%s/bin/linux/%s/kubelet" - KubeletURLPathTmplCN = "/release/%s/bin/linux/%s/kubelet" -) - -// Kubelet is a Binary for kubelet. -type Kubelet struct { - *Binary -} - -// NewKubelet returns a new Kubelet. -func NewKubelet(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch string) (*Kubelet, error) { - fileName := KubeletName - file, err := NewFile(Params{ - SSHClient: sshClient, - RootFs: rootFs, - Type: FileBinary, - Name: fileName, - LocalFullPath: filepath.Join(rootFs.ClusterRootFsDir(), KubeletID, version, arch, fileName), - RemoteFullPath: filepath.Join(BinDir, fileName), - }) - if err != nil { - return nil, err - } - - u := parseURL(DefaultDownloadHostGoogle, fmt.Sprintf(KubeletURLPathTmpl, version, arch)) - binary := NewBinary(BinaryParams{ - File: file, - ID: KubeletID, - Version: version, - Arch: arch, - URL: u, - }) - - return &Kubelet{binary}, nil -} - -// SetZone override Binary's SetZone method. -func (k *Kubelet) SetZone(zone string) { - if strings.EqualFold(zone, ZONE) { - k.SetHost(DefaultDownloadHostQingStor) - k.SetPath(fmt.Sprintf(KubeletURLPathTmplCN, k.version, k.arch)) - } -} diff --git a/pkg/service/operation/file/repositoryISO.go b/pkg/service/operation/file/repositoryISO.go deleted file mode 100644 index 61062851..00000000 --- a/pkg/service/operation/file/repositoryISO.go +++ /dev/null @@ -1,116 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package file - -import ( - "fmt" - "path/filepath" - "strings" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/rootfs" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/file/checksum" - "github.com/kubesphere/kubekey/v3/util/osrelease" -) - -// ISO info -const ( - ISOName = "%s.iso" - ISOID = "iso" - ISOURLPathTmpl = "/kubesphere/kubekey/releases/download/v2.2.2/%s" -) - -// ISO is a Binary for repository ISO file. -type ISO struct { - *Binary - HTTPChecksum *checksum.HTTPChecksum -} - -// NewISO returns a new repository ISO. -func NewISO(sshClient ssh.Interface, rootFs rootfs.Interface, os *osrelease.Data, arch string, isoName string) (*ISO, error) { - var ( - fileName string - urlPath string - ) - - if isoName == infrav1.AUTO { - fileName = generateISOName(os, arch) - if fileName == "" { - return nil, fmt.Errorf("" + - "can not detect the ISO file automatically, only support Ubuntu/Debian/CentOS. " + - "Please specify the ISO file name in the '.spec.repository.iso' field") - } - urlPath = fmt.Sprintf(ISOURLPathTmpl, fileName) - } else { - fileName = isoName - urlPath = fmt.Sprintf(ISOURLPathTmpl, fileName) - } - - file, err := NewFile(Params{ - SSHClient: sshClient, - RootFs: rootFs, - Type: FileBinary, - Name: fileName, - LocalFullPath: filepath.Join(rootFs.ClusterRootFsDir(), fileName), - RemoteFullPath: filepath.Join(MntDir, fileName), - }) - if err != nil { - return nil, err - } - - u := parseURL(DefaultDownloadHost, urlPath) - binary := NewBinary(BinaryParams{ - File: file, - ID: os.ID, - Version: os.VersionID, - Arch: arch, - URL: u, - }) - - checksumURL := parseURL(DefaultDownloadHost, fmt.Sprintf(ISOURLPathTmpl, generateCheckFileName(binary.Name()))) - httpChecksum := checksum.NewHTTPChecksum(checksumURL, binary.Name(), binary.file.rootFs) - binary.AppendChecksum(httpChecksum) - - return &ISO{binary, httpChecksum}, nil -} - -func generateISOName(os *osrelease.Data, arch string) string { - var fileName string - switch os.ID { - case osrelease.UbuntuID: - fileName = fmt.Sprintf(ISOName, strings.Join([]string{os.ID, os.VersionID, "debs", arch}, "-")) - case osrelease.DebianID: - fileName = fmt.Sprintf(ISOName, strings.Join([]string{os.ID + os.VersionID, "debs", arch}, "-")) - case osrelease.CentosID: - fileName = fmt.Sprintf(ISOName, strings.Join([]string{os.ID + os.VersionID, "rpms", arch}, "-")) - default: - fileName = fmt.Sprintf(ISOName, strings.Join([]string{os.ID, os.VersionID, arch}, "-")) - } - return fileName -} - -func generateCheckFileName(isoName string) string { - return isoName[0:strings.LastIndex(isoName, "-")] + ".iso.sha256sum.txt" -} - -// SetZone override Binary's SetZone method. -func (i *ISO) SetZone(zone string) { - if strings.EqualFold(zone, ZONE) { - return - } -} diff --git a/pkg/service/operation/file/repositoryISO_test.go b/pkg/service/operation/file/repositoryISO_test.go deleted file mode 100644 index f43832bf..00000000 --- a/pkg/service/operation/file/repositoryISO_test.go +++ /dev/null @@ -1,44 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package file - -import ( - "testing" -) - -func Test_generateCheckFileName(t *testing.T) { - tests := []struct { - isoName string - want string - }{ - { - isoName: "centos7-rpms-amd64.iso", - want: "centos7-rpms.iso.sha256sum.txt", - }, - { - isoName: "ubuntu-22.04-debs-arm64.iso", - want: "ubuntu-22.04-debs.iso.sha256sum.txt", - }, - } - for _, tt := range tests { - t.Run("", func(t *testing.T) { - if got := generateCheckFileName(tt.isoName); got != tt.want { - t.Errorf("generateCheckFileName() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/service/operation/file/runc.go b/pkg/service/operation/file/runc.go deleted file mode 100644 index 4e766774..00000000 --- a/pkg/service/operation/file/runc.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package file - -import ( - "fmt" - "path/filepath" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/rootfs" -) - -// runc info -const ( - RuncName = "runc.%s" - RuncID = "runc" - RuncURLPathTmpl = "/opencontainers/runc/releases/download/%s/runc.%s" - RuncDefaultVersion = "v1.1.1" -) - -// Runc is a Binary for runc. -type Runc struct { - *Binary -} - -// NewRunc returns a new Runc. -func NewRunc(sshClient ssh.Interface, rootFs rootfs.Interface, version, arch string) (*Runc, error) { - fileName := fmt.Sprintf(RuncName, arch) - file, err := NewFile(Params{ - SSHClient: sshClient, - RootFs: rootFs, - Type: FileBinary, - Name: fileName, - LocalFullPath: filepath.Join(rootFs.ClusterRootFsDir(), RuncID, version, arch, fileName), - RemoteFullPath: filepath.Join(BinDir, fileName), - }) - if err != nil { - return nil, err - } - - u := parseURL(DefaultDownloadHost, fmt.Sprintf(RuncURLPathTmpl, version, arch)) - binary := NewBinary(BinaryParams{ - File: file, - ID: RuncID, - Version: version, - Arch: arch, - URL: u, - }) - - return &Runc{binary}, nil -} diff --git a/pkg/service/operation/file/template.go b/pkg/service/operation/file/template.go deleted file mode 100644 index 500a5ce0..00000000 --- a/pkg/service/operation/file/template.go +++ /dev/null @@ -1,79 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package file - -import ( - "os" - "path/filepath" - "text/template" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/rootfs" - "github.com/kubesphere/kubekey/v3/pkg/util/filesystem" -) - -// Data is the data that will be passed to the template. -type Data map[string]interface{} - -// Template is an implementation of the Template interface. -type Template struct { - *File - template *template.Template - data Data - dst string -} - -// NewTemplate returns a new Template. -func NewTemplate(sshClient ssh.Interface, rootFs rootfs.Interface, template *template.Template, data Data, dst string) (*Template, error) { - file, err := NewFile(Params{ - SSHClient: sshClient, - RootFs: rootFs, - Name: template.Name(), - Type: FileTemplate, - LocalFullPath: filepath.Join(rootFs.HostRootFsDir(sshClient.Host()), template.Name()), - RemoteFullPath: dst, - }) - if err != nil { - return nil, err - } - return &Template{ - file, - template, - data, - dst, - }, nil -} - -// RenderToLocal renders the template to the local filesystem. -func (t *Template) RenderToLocal() error { - dir := filepath.Dir(t.localFullPath) - if _, err := os.Stat(dir); os.IsNotExist(err) { - if err = os.MkdirAll(dir, filesystem.FileMode0755); err != nil { - return err - } - } - - f, err := os.Create(t.localFullPath) - if err != nil { - return err - } - - if err := t.template.Execute(f, t.data); err != nil { - return err - } - return nil -} diff --git a/pkg/service/operation/interface.go b/pkg/service/operation/interface.go deleted file mode 100644 index 0edd3e47..00000000 --- a/pkg/service/operation/interface.go +++ /dev/null @@ -1,81 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package operation define the remote instance operations interface. -package operation - -import ( - "net/url" - "time" - - "github.com/kubesphere/kubekey/v3/pkg/service/operation/file" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/file/checksum" -) - -// File interface defines the operations for normal file which needed to be copied to remote. -type File interface { - Name() string - Type() file.Type - LocalPath() string - RemotePath() string - LocalExist() bool - RemoteExist() bool - Copy(override bool) error - Fetch(override bool) error - Chmod(option string) error -} - -// Binary interface defines the operations for Kubernetes needed binaries which usually needed to be copied to remote. -type Binary interface { - File - ID() string - Arch() string - Version() string - URL() *url.URL - SetURL(url string) - SetHost(host string) - SetPath(path string) - SetZone(zone string) - AppendChecksum(c checksum.Interface) - Get(timeout time.Duration) error - CompareChecksum() error -} - -// Template interface defines the operations for Kubernetes needed template files (systemd files, config files .e.g) -// which usually needed to be copied to remote. -type Template interface { - File - RenderToLocal() error -} - -// User interface defines the operations for remote instance Linux user. -type User interface { - Add() error -} - -// Directory interface defines the operations for remote instance Linux directory. -type Directory interface { - Make() error - Chown(user string) error - Remove() error -} - -// Repository interface defines the operations for remote instance Linux repository. -type Repository interface { - Update() error - Install(pkg ...string) error - Add(path string) error -} diff --git a/pkg/service/operation/repository/doc.go b/pkg/service/operation/repository/doc.go deleted file mode 100644 index 64ab71bf..00000000 --- a/pkg/service/operation/repository/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package repository defines the operations on the remote instance linux software packages repository. -package repository diff --git a/pkg/service/operation/repository/repository_deb.go b/pkg/service/operation/repository/repository_deb.go deleted file mode 100644 index a7255e15..00000000 --- a/pkg/service/operation/repository/repository_deb.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package repository - -import ( - "fmt" - "strings" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" -) - -// Debian is a repository manager implementation for Debian. -type Debian struct { - SSHClient ssh.Interface -} - -// NewDeb returns a new Debian repository manager. -func NewDeb(sshClient ssh.Interface) *Debian { - return &Debian{ - SSHClient: sshClient, - } -} - -// Add adds a local repository using the iso file. -func (d *Debian) Add(path string) error { - if _, err := d.SSHClient.SudoCmd( - fmt.Sprintf("echo 'deb [trusted=yes] file://%s ./' "+ - "| sudo tee /etc/apt/sources.list.d/kubekey.list > /dev/null", path)); err != nil { - return err - } - return nil -} - -// Update updates the repository cache. -func (d *Debian) Update() error { - if _, err := d.SSHClient.Cmd("sudo apt-get update"); err != nil { - return err - } - return nil -} - -// Install installs common packages. -func (d *Debian) Install(pkg ...string) error { - if len(pkg) == 0 { - pkg = []string{"socat", "conntrack", "ipset", "ebtables", "chrony", "ipvsadm"} - } - if _, err := d.SSHClient.SudoCmdf("apt install -y %s", strings.Join(pkg, " ")); err != nil { - return err - } - return nil -} diff --git a/pkg/service/operation/repository/repository_rpm.go b/pkg/service/operation/repository/repository_rpm.go deleted file mode 100644 index 3af8a8d2..00000000 --- a/pkg/service/operation/repository/repository_rpm.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package repository - -import ( - "fmt" - "strings" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" -) - -// RedhatPackageManager is a repository manager implementation for Redhat, Centos. -type RedhatPackageManager struct { - SSHClient ssh.Interface -} - -// NewRPM returns a new RedhatPackageManager. -func NewRPM(sshClient ssh.Interface) *RedhatPackageManager { - return &RedhatPackageManager{ - SSHClient: sshClient, - } -} - -// Add adds a local repository using the iso file. -func (r *RedhatPackageManager) Add(path string) error { - content := fmt.Sprintf(`cat << EOF > /etc/yum.repos.d/kubekey.repo -[base-local] -name=KubeKey-local -baseurl=file://%s -enabled=1 -gpgcheck=0 -EOF -`, path) - - if _, err := r.SSHClient.SudoCmd(content); err != nil { - return err - } - return nil -} - -// Update updates the repository cache. -func (r *RedhatPackageManager) Update() error { - if _, err := r.SSHClient.SudoCmd("yum clean all && yum makecache"); err != nil { - return err - } - return nil -} - -// Install installs common packages. -func (r *RedhatPackageManager) Install(pkg ...string) error { - if len(pkg) == 0 { - pkg = []string{"openssl", "socat", "conntrack", "ipset", "ebtables", "chrony", "ipvsadm"} - } - if _, err := r.SSHClient.SudoCmdf("yum install -y %s", strings.Join(pkg, " ")); err != nil { - return err - } - return nil -} diff --git a/pkg/service/operation/repository/service.go b/pkg/service/operation/repository/service.go deleted file mode 100644 index 14d4a410..00000000 --- a/pkg/service/operation/repository/service.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package repository - -import ( - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/util/osrelease" -) - -// Service holds a collection of interfaces. -// The interfaces are broken down like this to group functions together. -type Service interface { - Add(path string) error - Update() error - Install(pkg ...string) error -} - -// NewService returns a new service given the remote instance package manager client. -func NewService(sshClient ssh.Interface, os *osrelease.Data) Service { - if os == nil { - return nil - } - - switch { - case os.IsLikeDebian(): - return NewDeb(sshClient) - case os.IsLikeFedora(): - return NewRPM(sshClient) - default: - return nil - } -} diff --git a/pkg/service/operation/user/doc.go b/pkg/service/operation/user/doc.go deleted file mode 100644 index cad7793d..00000000 --- a/pkg/service/operation/user/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package user defines the user operations on the remote instance. -package user diff --git a/pkg/service/operation/user/service.go b/pkg/service/operation/user/service.go deleted file mode 100644 index c545aaa0..00000000 --- a/pkg/service/operation/user/service.go +++ /dev/null @@ -1,38 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package user - -import ( - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" -) - -// Service holds a collection of interfaces. -// The interfaces are broken down like this to group functions together. -type Service struct { - SSHClient ssh.Interface - Name string - Desc string -} - -// NewService returns a new service given the remote instance Linux user. -func NewService(sshClient ssh.Interface, name, desc string) *Service { - return &Service{ - SSHClient: sshClient, - Name: name, - Desc: desc, - } -} diff --git a/pkg/service/operation/user/user.go b/pkg/service/operation/user/user.go deleted file mode 100644 index d1e6385b..00000000 --- a/pkg/service/operation/user/user.go +++ /dev/null @@ -1,32 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package user - -import ( - "fmt" - - "github.com/pkg/errors" -) - -// Add adds a new Linux user to the remote instance. -func (s *Service) Add() error { - _, err := s.SSHClient.SudoCmd(fmt.Sprintf("useradd -M -c '%s' -s /sbin/nologin -r %s || :", s.Desc, s.Name)) - if err != nil { - return errors.Wrapf(err, "failed to add user %s", s.Name) - } - return nil -} diff --git a/pkg/service/provisioning/cloudinit/action.go b/pkg/service/provisioning/cloudinit/action.go deleted file mode 100644 index 00bdee6d..00000000 --- a/pkg/service/provisioning/cloudinit/action.go +++ /dev/null @@ -1,57 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/service/provisioning/commands" -) - -const ( - // Supported cloud config modules. - writefiles = "write_files" - runcmd = "runcmd" -) - -type action interface { - Unmarshal(userData []byte) error - Commands() ([]commands.Cmd, error) - Run() error -} - -type actionFactory struct { - sshClient ssh.Interface -} - -// newActionFactory returns a new action factory. -func newActionFactory(sshClient ssh.Interface) *actionFactory { - return &actionFactory{ - sshClient: sshClient, - } -} - -func (a *actionFactory) action(name string) action { - switch name { - case writefiles: - return newWriteFilesAction(a.sshClient) - case runcmd: - return newRunCmdAction() - default: - // TODO Add a logger during the refactor and log this unknown module - return newUnknown(name) - } -} diff --git a/pkg/service/provisioning/cloudinit/adapter.go b/pkg/service/provisioning/cloudinit/adapter.go deleted file mode 100644 index 2d1f9dcc..00000000 --- a/pkg/service/provisioning/cloudinit/adapter.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "bufio" - "bytes" - "regexp" - "strings" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" -) - -// getActions parses the cloud config yaml into a slice of actions to run. -// Parsing manually is required because the order of the cloud config's actions must be maintained. -func getActions(sshClient ssh.Interface, userData []byte) ([]action, error) { - actionRegEx := regexp.MustCompile(`^[a-zA-Z_]*:`) - lines := make([]string, 0) - actions := make([]action, 0) - actionFactory := newActionFactory(sshClient) - - var act action - - // scans the file searching for keys/top level actions. - scanner := bufio.NewScanner(bytes.NewReader(userData)) - for scanner.Scan() { - line := scanner.Text() - // if the line is key/top level action - if actionRegEx.MatchString(line) { - // converts the file fragment scanned up to now into the current action, if any - if act != nil { - actionBlock := strings.Join(lines, "\n") - if err := act.Unmarshal([]byte(actionBlock)); err != nil { - return nil, errors.WithStack(err) - } - actions = append(actions, act) - lines = lines[:0] - } - - // creates the new action - actionName := strings.TrimSuffix(line, ":") - act = actionFactory.action(actionName) - } - - lines = append(lines, line) - } - - // converts the last file fragment scanned into the current action, if any - if act != nil { - actionBlock := strings.Join(lines, "\n") - if err := act.Unmarshal([]byte(actionBlock)); err != nil { - return nil, errors.WithStack(err) - } - actions = append(actions, act) - } - - return actions, scanner.Err() -} diff --git a/pkg/service/provisioning/cloudinit/doc.go b/pkg/service/provisioning/cloudinit/doc.go deleted file mode 100644 index d01cf41c..00000000 --- a/pkg/service/provisioning/cloudinit/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package cloudinit defines cloud init adapter for existing nodes. -package cloudinit diff --git a/pkg/service/provisioning/cloudinit/runcmd.go b/pkg/service/provisioning/cloudinit/runcmd.go deleted file mode 100644 index 249d2f60..00000000 --- a/pkg/service/provisioning/cloudinit/runcmd.go +++ /dev/null @@ -1,80 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "strings" - - "github.com/pkg/errors" - "sigs.k8s.io/yaml" - - "github.com/kubesphere/kubekey/v3/pkg/service/provisioning/commands" -) - -// runCmd defines parameters of a shell command that is equivalent to an action found in the cloud init rundcmd module. -type runCmd struct { - Cmds []commands.Cmd `json:"runcmd,"` -} - -func newRunCmdAction() action { - return &runCmd{} -} - -// Unmarshal the runCmd. -func (a *runCmd) Unmarshal(userData []byte) error { - if err := yaml.Unmarshal(userData, a); err != nil { - return errors.Wrapf(err, "error parsing run_cmd action: %s", userData) - } - return nil -} - -// Commands returns the commands. -func (a *runCmd) Commands() ([]commands.Cmd, error) { - cmds := make([]commands.Cmd, 0) - for _, c := range a.Cmds { - // requires to ignore some errors, and this requires to modify the cmd generate by CABPK by default... - c = hackKubeadmIgnoreErrors(c) - cmds = append(cmds, c) - } - return cmds, nil -} - -// Run runs the commands. -func (a *runCmd) Run() error { - return nil -} - -func hackKubeadmIgnoreErrors(c commands.Cmd) commands.Cmd { - // case kubeadm commands are defined as a string - if c.Cmd == "/bin/sh" && len(c.Args) >= 2 { - if c.Args[0] == "-c" { - c.Args[1] = strings.Replace(c.Args[1], "kubeadm init", "kubeadm init --ignore-preflight-errors=all", 1) - c.Args[1] = strings.Replace(c.Args[1], "kubeadm join", "kubeadm join --ignore-preflight-errors=all", 1) - } - } - - // case kubeadm commands are defined as a list - if c.Cmd == "kubeadm" && len(c.Args) >= 1 { - if c.Args[0] == "init" || c.Args[0] == "join" { - c.Args = append(c.Args, "") // make space - copy(c.Args[2:], c.Args[1:]) // shift elements - c.Args[1] = "--ignore-preflight-errors=all" // insert the additional arg - } - } - - return c -} diff --git a/pkg/service/provisioning/cloudinit/runcmd_test.go b/pkg/service/provisioning/cloudinit/runcmd_test.go deleted file mode 100644 index c2d77400..00000000 --- a/pkg/service/provisioning/cloudinit/runcmd_test.go +++ /dev/null @@ -1,76 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "testing" - - . "github.com/onsi/gomega" - - "github.com/kubesphere/kubekey/v3/pkg/service/provisioning/commands" -) - -func TestRunCmdUnmarshal(t *testing.T) { - g := NewWithT(t) - - cloudData := ` -runcmd: -- [ ls, -l, / ] -- "ls -l /"` - r := runCmd{} - err := r.Unmarshal([]byte(cloudData)) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(r.Cmds).To(HaveLen(2)) - - expected0 := commands.Cmd{Cmd: "ls", Args: []string{"-l", "/"}} - g.Expect(r.Cmds[0]).To(Equal(expected0)) - - expected1 := commands.Cmd{Cmd: "/bin/sh", Args: []string{"-c", "ls -l /"}} - g.Expect(r.Cmds[1]).To(Equal(expected1)) -} - -func TestHackKubeadmIgnoreErrors(t *testing.T) { - g := NewWithT(t) - - cloudData := ` -runcmd: -- kubeadm init --config=/run/kubeadm/kubeadm.yaml -- [ kubeadm, join, --config=/run/kubeadm/kubeadm-controlplane-join-config.yaml ] -- kubeadm init --ignore-preflight-errors=all --config=/run/kubeadm/kubeadm.yaml -- [ kubeadm, join, --ignore-preflight-errors=all, --config=/run/kubeadm/kubeadm-controlplane-join-config.yaml ]` - - r := runCmd{} - err := r.Unmarshal([]byte(cloudData)) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(r.Cmds).To(HaveLen(4)) - - r.Cmds[0] = hackKubeadmIgnoreErrors(r.Cmds[0]) - expected0 := commands.Cmd{Cmd: "/bin/sh", Args: []string{"-c", "kubeadm init --ignore-preflight-errors=all --config=/run/kubeadm/kubeadm.yaml"}} - g.Expect(r.Cmds[0]).To(Equal(expected0)) - - r.Cmds[1] = hackKubeadmIgnoreErrors(r.Cmds[1]) - expected1 := commands.Cmd{Cmd: "kubeadm", Args: []string{"join", "--ignore-preflight-errors=all", "--config=/run/kubeadm/kubeadm-controlplane-join-config.yaml"}} - g.Expect(r.Cmds[1]).To(Equal(expected1)) - - r.Cmds[2] = hackKubeadmIgnoreErrors(r.Cmds[2]) - expected2 := commands.Cmd{Cmd: "/bin/sh", Args: []string{"-c", "kubeadm init --ignore-preflight-errors=all --config=/run/kubeadm/kubeadm.yaml"}} - g.Expect(r.Cmds[0]).To(Equal(expected2)) - - r.Cmds[3] = hackKubeadmIgnoreErrors(r.Cmds[3]) - expected3 := commands.Cmd{Cmd: "kubeadm", Args: []string{"join", "--ignore-preflight-errors=all", "--config=/run/kubeadm/kubeadm-controlplane-join-config.yaml"}} - g.Expect(r.Cmds[1]).To(Equal(expected3)) -} diff --git a/pkg/service/provisioning/cloudinit/service.go b/pkg/service/provisioning/cloudinit/service.go deleted file mode 100644 index 627c6fd8..00000000 --- a/pkg/service/provisioning/cloudinit/service.go +++ /dev/null @@ -1,63 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "github.com/pkg/errors" - "sigs.k8s.io/yaml" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/service/provisioning/commands" -) - -// Service holds a collection of interfaces. -// The interfaces are broken down like this to group functions together. -type Service struct { - SSHClient ssh.Interface -} - -// NewService returns a new service. -func NewService(sshClient ssh.Interface) *Service { - return &Service{ - SSHClient: sshClient, - } -} - -// RawBootstrapDataToProvisioningCommands converts raw bootstrap data to provisioning commands. -func (s *Service) RawBootstrapDataToProvisioningCommands(config []byte) ([]commands.Cmd, error) { - // validate cloudConfigScript is a valid yaml, as required by the cloud config specification - if err := yaml.Unmarshal(config, &map[string]interface{}{}); err != nil { - return nil, errors.Wrapf(err, "cloud-config is not valid yaml") - } - - // parse the cloud config yaml into a slice of cloud config actions. - actions, err := getActions(s.SSHClient, config) - if err != nil { - return nil, err - } - - commands := []commands.Cmd{} - for _, action := range actions { - cmds, err := action.Commands() - if err != nil { - return commands, err - } - commands = append(commands, cmds...) - } - - return commands, nil -} diff --git a/pkg/service/provisioning/cloudinit/unknown.go b/pkg/service/provisioning/cloudinit/unknown.go deleted file mode 100644 index 8a3b5179..00000000 --- a/pkg/service/provisioning/cloudinit/unknown.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cloudinit - -import ( - "encoding/json" - - "github.com/pkg/errors" - - "github.com/kubesphere/kubekey/v3/pkg/service/provisioning/commands" -) - -type unknown struct { - module string - lines []string -} - -func newUnknown(module string) action { - return &unknown{module: module} -} - -// Unmarshal will unmarshal unknown actions and slurp the value. -func (u *unknown) Unmarshal(data []byte) error { - // try unmarshalling to a slice of strings - var s1 []string - if err := json.Unmarshal(data, &s1); err != nil { - if _, ok := err.(*json.UnmarshalTypeError); !ok { - return errors.WithStack(err) - } - } else { - u.lines = s1 - return nil - } - - // If it's not a slice of strings it should be one string value - var s2 string - if err := json.Unmarshal(data, &s2); err != nil { - return errors.WithStack(err) - } - - u.lines = []string{s2} - return nil -} - -// Commands returns the commands to run for the unknown action. -func (u *unknown) Commands() ([]commands.Cmd, error) { - return []commands.Cmd{}, nil -} - -// Run runs the commands for the unknown action. -func (u *unknown) Run() error { - return nil -} diff --git a/pkg/service/provisioning/cloudinit/unknown_test.go b/pkg/service/provisioning/cloudinit/unknown_test.go deleted file mode 100644 index 458c80b8..00000000 --- a/pkg/service/provisioning/cloudinit/unknown_test.go +++ /dev/null @@ -1,45 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "testing" - - . "github.com/onsi/gomega" -) - -func TestUnknown_Run(t *testing.T) { - g := NewWithT(t) - - u := &unknown{ - lines: []string{}, - } - lines, err := u.Commands() - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(lines).To(HaveLen(0)) -} - -func TestUnknown_Unmarshal(t *testing.T) { - g := NewWithT(t) - - u := &unknown{} - expected := []string{"test 1", "test 2", "test 3"} - input := `["test 1", "test 2", "test 3"]` - - g.Expect(u.Unmarshal([]byte(input))).To(Succeed()) - g.Expect(u.lines).To(Equal(expected)) -} diff --git a/pkg/service/provisioning/cloudinit/writefiles.go b/pkg/service/provisioning/cloudinit/writefiles.go deleted file mode 100644 index 6cd1bd8c..00000000 --- a/pkg/service/provisioning/cloudinit/writefiles.go +++ /dev/null @@ -1,244 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "bytes" - "compress/gzip" - "encoding/base64" - "fmt" - "io" - "os" - "path/filepath" - "strings" - - "github.com/pkg/errors" - "sigs.k8s.io/yaml" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/service/operation" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/directory" - "github.com/kubesphere/kubekey/v3/pkg/service/provisioning/commands" - "github.com/kubesphere/kubekey/v3/pkg/util/filesystem" -) - -const ( - kubeadmInitPath = "/run/kubeadm/kubeadm.yaml" - kubeproxyComponentConfig = ` ---- -apiVersion: kubeproxy.config.k8s.io/v1alpha1 -kind: KubeProxyConfiguration -conntrack: -# Skip setting sysctl value "net.netfilter.nf_conntrack_max" -# It is a global variable that affects other namespaces - maxPerCore: 0 -` -) - -// writeFilesAction defines a list of files that should be written to a node. -type writeFilesAction struct { - sshClient ssh.Interface - directoryFactory func(sshClient ssh.Interface, path string, mode os.FileMode) operation.Directory - - Files []files `json:"write_files,"` -} - -type files struct { - Path string `json:"path,"` - Encoding string `json:"encoding,omitempty"` - Owner string `json:"owner,omitempty"` - Permissions string `json:"permissions,omitempty"` - Content string `json:"content,"` - Append bool `json:"append,"` -} - -func newWriteFilesAction(sshClient ssh.Interface) action { - return &writeFilesAction{ - sshClient: sshClient, - } -} - -func (a *writeFilesAction) getDirectoryService(path string, mode os.FileMode) operation.Directory { - if a.directoryFactory != nil { - return a.directoryFactory(a.sshClient, path, mode) - } - return directory.NewService(a.sshClient, path, mode) -} - -// Unmarshal unmarshals the given content into the given encoding. -func (a *writeFilesAction) Unmarshal(userData []byte) error { - if err := yaml.Unmarshal(userData, a); err != nil { - return errors.Wrapf(err, "error parsing write_files action: %s", userData) - } - return nil -} - -// Commands return a list of commands to run on the node. -// Each command defines the parameters of a shell command necessary to generate a file replicating the cloud-init write_files module. -func (a *writeFilesAction) Commands() ([]commands.Cmd, error) { - cmds := make([]commands.Cmd, 0) - for _, f := range a.Files { - // Fix attributes and apply defaults - path := fixPath(f.Path) // NB. the real cloud init module for writes files converts path into absolute paths; this is not possible here... - encodings := fixEncoding(f.Encoding) - owner := fixOwner(f.Owner) - permissions := fixPermissions(f.Permissions) - content, err := fixContent(f.Content, encodings) - if path == kubeadmInitPath { - content += kubeproxyComponentConfig - } - if err != nil { - return cmds, errors.Wrapf(err, "error decoding content for %s", path) - } - - // Make the directory so cat + redirection will work - directory := filepath.Dir(path) - cmds = append(cmds, commands.Cmd{Cmd: "mkdir", Args: []string{"-p", directory}}) - - redirects := ">" - if f.Append { - redirects = ">>" - } - - // generate a command that will create a file with the expected contents. - cmds = append(cmds, commands.Cmd{Cmd: "/bin/sh", Args: []string{"-c", fmt.Sprintf("echo '%s' %s %s", content, redirects, path)}}) - - // if permissions are different than default ownership, add a command to modify the permissions. - if permissions != "0644" { - cmds = append(cmds, commands.Cmd{Cmd: "chmod", Args: []string{permissions, path}}) - } - - // if ownership is different than default ownership, add a command to modify file ownerhsip. - if owner != "root:root" { - cmds = append(cmds, commands.Cmd{Cmd: "chown", Args: []string{owner, path}}) - } - } - return cmds, nil -} - -// Run runs the action. -func (a *writeFilesAction) Run() error { - for _, f := range a.Files { - // Fix attributes and apply defaults - path := fixPath(f.Path) // NB. the real cloud init module for writes files converts path into absolute paths; this is not possible here... - encodings := fixEncoding(f.Encoding) - content, err := fixContent(f.Content, encodings) - if path == kubeadmInitPath { - content += kubeproxyComponentConfig - } - if err != nil { - return errors.Wrapf(err, "error decoding content for %s", path) - } - - // Make the directory so cat + redirection will work - dir := filepath.Dir(path) - svc := a.getDirectoryService(dir, os.FileMode(filesystem.FileMode0755)) - if err := svc.Make(); err != nil { - return err - } - if err := svc.Chown("root"); err != nil { - return err - } - - redirects := ">" - if f.Append { - redirects = ">>" - } - if _, err := a.sshClient.SudoCmdf("echo '%s' %s %s", content, redirects, path); err != nil { - return err - } - } - return nil -} - -func fixPath(p string) string { - return strings.TrimSpace(p) -} - -func fixOwner(o string) string { - o = strings.TrimSpace(o) - if o != "" { - return o - } - return "root:root" -} - -func fixPermissions(p string) string { - p = strings.TrimSpace(p) - if p != "" { - return p - } - return "0644" -} - -func fixEncoding(e string) []string { - e = strings.ToLower(e) - e = strings.TrimSpace(e) - - switch e { - case "gz", "gzip": - return []string{"application/x-gzip"} - case "gz+base64", "gzip+base64", "gz+b64", "gzip+b64": - return []string{"application/base64", "application/x-gzip"} - case "base64", "b64": - return []string{"application/base64"} - } - - return []string{"text/plain"} -} - -func fixContent(content string, encodings []string) (string, error) { - for _, e := range encodings { - switch e { - case "application/base64": - rByte, err := base64.StdEncoding.DecodeString(content) - if err != nil { - return content, errors.WithStack(err) - } - return string(rByte), nil - case "application/x-gzip": - rByte, err := gUnzipData([]byte(content)) - if err != nil { - return content, err - } - return string(rByte), nil - case "text/plain": - return content, nil - default: - return content, errors.Errorf("Unknown bootstrap data encoding: %q", content) - } - } - return content, nil -} - -func gUnzipData(data []byte) ([]byte, error) { - var r io.Reader - var err error - b := bytes.NewBuffer(data) - r, err = gzip.NewReader(b) - if err != nil { - return nil, errors.WithStack(err) - } - - var resB bytes.Buffer - _, err = resB.ReadFrom(r) - if err != nil { - return nil, errors.WithStack(err) - } - - return resB.Bytes(), nil -} diff --git a/pkg/service/provisioning/cloudinit/writefiles_test.go b/pkg/service/provisioning/cloudinit/writefiles_test.go deleted file mode 100644 index b378922e..00000000 --- a/pkg/service/provisioning/cloudinit/writefiles_test.go +++ /dev/null @@ -1,173 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cloudinit - -import ( - "bytes" - "compress/gzip" - "testing" - - . "github.com/onsi/gomega" - - "github.com/kubesphere/kubekey/v3/pkg/service/provisioning/commands" -) - -func TestWriteFiles(t *testing.T) { - var useCases = []struct { - name string - w writeFilesAction - expectedCmds []commands.Cmd - }{ - { - name: "two files pass", - w: writeFilesAction{ - Files: []files{ - {Path: "foo", Content: "bar"}, - {Path: "baz", Content: "qux"}, - }, - }, - expectedCmds: []commands.Cmd{ - {Cmd: "mkdir", Args: []string{"-p", "."}}, - {Cmd: "/bin/sh", Args: []string{"-c", "echo 'bar' > foo"}}, - {Cmd: "mkdir", Args: []string{"-p", "."}}, - {Cmd: "/bin/sh", Args: []string{"-c", "echo 'qux' > baz"}}, - }, - }, - { - name: "owner different than default", - w: writeFilesAction{ - Files: []files{ - {Path: "foo", Content: "bar", Owner: "baz:baz"}, - }, - }, - expectedCmds: []commands.Cmd{ - {Cmd: "mkdir", Args: []string{"-p", "."}}, - {Cmd: "/bin/sh", Args: []string{"-c", "echo 'bar' > foo"}}, - {Cmd: "chown", Args: []string{"baz:baz", "foo"}}, - }, - }, - { - name: "permissions different than default", - w: writeFilesAction{ - Files: []files{ - {Path: "foo", Content: "bar", Permissions: "755"}, - }, - }, - expectedCmds: []commands.Cmd{ - {Cmd: "mkdir", Args: []string{"-p", "."}}, - {Cmd: "/bin/sh", Args: []string{"-c", "echo 'bar' > foo"}}, - {Cmd: "chmod", Args: []string{"755", "foo"}}, - }, - }, - { - name: "append", - w: writeFilesAction{ - Files: []files{ - {Path: "foo", Content: "bar", Append: true}, - }, - }, - expectedCmds: []commands.Cmd{ - {Cmd: "mkdir", Args: []string{"-p", "."}}, - {Cmd: "/bin/sh", Args: []string{"-c", "echo 'bar' >> foo"}}, - }, - }, - } - - for _, rt := range useCases { - t.Run(rt.name, func(t *testing.T) { - g := NewWithT(t) - - cmds, err := rt.w.Commands() - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(rt.expectedCmds).To(Equal(cmds)) - }) - } -} - -func TestFixContent(t *testing.T) { - v := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - gv, _ := gZipData([]byte(v)) - var useCases = []struct { - name string - content string - encoding string - expectedContent string - expectedError bool - }{ - { - name: "plain text", - content: "foobar", - expectedContent: "foobar", - }, - { - name: "base64 data", - content: "YWJjMTIzIT8kKiYoKSctPUB+", - encoding: "base64", - expectedContent: "abc123!?$*&()'-=@~", - }, - { - name: "gzip data", - content: string(gv), - encoding: "gzip", - expectedContent: v, - }, - } - - for _, rt := range useCases { - t.Run(rt.name, func(t *testing.T) { - g := NewWithT(t) - - encoding := fixEncoding(rt.encoding) - c, err := fixContent(rt.content, encoding) - if rt.expectedError { - g.Expect(err).To(HaveOccurred()) - } else { - g.Expect(err).NotTo(HaveOccurred()) - } - - g.Expect(rt.expectedContent).To(Equal(c)) - }) - } -} - -func TestUnzipData(t *testing.T) { - g := NewWithT(t) - - value := []byte("foobarbazquxfoobarbazquxfoobarbazquxfoobarbazquxfoobarbazquxfoobarbazquxfoobarbazquxfoobarbazquxfoobarbazqux") - gvalue, _ := gZipData(value) - dvalue, _ := gUnzipData(gvalue) - g.Expect(value).To(Equal(dvalue)) -} - -func gZipData(data []byte) ([]byte, error) { - var b bytes.Buffer - gz := gzip.NewWriter(&b) - - if _, err := gz.Write(data); err != nil { - return nil, err - } - - if err := gz.Flush(); err != nil { - return nil, err - } - - if err := gz.Close(); err != nil { - return nil, err - } - - return b.Bytes(), nil -} diff --git a/pkg/service/provisioning/commands/commands.go b/pkg/service/provisioning/commands/commands.go deleted file mode 100644 index cf0f4f3b..00000000 --- a/pkg/service/provisioning/commands/commands.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package commands deals with various machine initialization methods viz. cloud-init, Ignition, -// etc. -package commands - -import ( - "encoding/json" - "strings" - - "github.com/pkg/errors" -) - -// Cmd defines a shell command. -type Cmd struct { - Cmd string - Args []string - Stdin string -} - -// UnmarshalJSON a runcmd command -// It can be either a list or a string. -// If the item is a list, the head of the list is the command and the tail are the args. -// If the item is a string, the whole command will be wrapped in `/bin/sh -c`. -func (c *Cmd) UnmarshalJSON(data []byte) error { - // First, try to decode the input as a list - var s1 []string - if err := json.Unmarshal(data, &s1); err != nil { - if _, ok := err.(*json.UnmarshalTypeError); !ok { - return errors.WithStack(err) - } - } else { - c.Cmd = s1[0] - c.Args = s1[1:] - return nil - } - - // If it's not a list, it must be a string - var s2 string - if err := json.Unmarshal(data, &s2); err != nil { - return errors.WithStack(err) - } - - c.Cmd = "/bin/sh" - c.Args = []string{"-c", s2} - - return nil -} - -// String returns the command as a string. -func (c *Cmd) String() string { - cmd := strings.Join(append([]string{c.Cmd}, c.Args...), " ") - if strings.HasPrefix(cmd, "/bin/sh -c") { - cmd = strings.TrimPrefix(cmd, "/bin/sh -c") - } else { - cmd = strings.TrimPrefix(cmd, "/bin/bash -c") - } - - return cmd -} diff --git a/pkg/service/provisioning/service.go b/pkg/service/provisioning/service.go deleted file mode 100644 index 2824ae9b..00000000 --- a/pkg/service/provisioning/service.go +++ /dev/null @@ -1,42 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package provisioning defines the provisioning operations on the remote instance generated by cloudinit or ignition. -package provisioning - -import ( - bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/service/provisioning/cloudinit" - "github.com/kubesphere/kubekey/v3/pkg/service/provisioning/commands" -) - -// Service holds a collection of interfaces. -// The interfaces are broken down like this to group functions together. -type Service interface { - RawBootstrapDataToProvisioningCommands(config []byte) ([]commands.Cmd, error) -} - -// NewService returns a new service given the cloud config format client. -func NewService(sshClient ssh.Interface, format bootstrapv1.Format) Service { - switch format { - case bootstrapv1.CloudConfig: - return cloudinit.NewService(sshClient) - default: - return cloudinit.NewService(sshClient) - } -} diff --git a/pkg/service/repository/doc.go b/pkg/service/repository/doc.go deleted file mode 100644 index d21b3f96..00000000 --- a/pkg/service/repository/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package repository contains the repository service. -package repository diff --git a/pkg/service/repository/repository.go b/pkg/service/repository/repository.go deleted file mode 100644 index 863dd479..00000000 --- a/pkg/service/repository/repository.go +++ /dev/null @@ -1,162 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package repository - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "github.com/pkg/errors" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/file" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/repository" - "github.com/kubesphere/kubekey/v3/pkg/service/util" - "github.com/kubesphere/kubekey/v3/pkg/util/filesystem" - "github.com/kubesphere/kubekey/v3/util/osrelease" -) - -// Check checks the OS release info. -func (s *Service) Check() error { - if !s.instanceScope.RepositoryEnabled() { - return nil - } - - output, err := s.sshClient.SudoCmd("cat /etc/os-release") - if err != nil { - return errors.Wrap(err, "failed to get os release") - } - - s.scope.V(4).Info("Get os release", "output", output) - osrData := osrelease.Parse(output) - if osrData == nil || osrData.ID == "" || osrData.VersionID == "" { - return errors.Errorf("failed to parse os release: %s", output) - } - s.os = osrData - return nil -} - -// Get gets the binary of ISO file and copy it to the remote instance. -func (s *Service) Get(timeout time.Duration) error { - if !s.instanceScope.RepositoryUseISO() { - return nil - } - - s.scope.V(4).Info("os release", "os", s.os) - iso, err := s.getISOService(s.os, s.instanceScope.Arch(), s.instanceScope.Repository().ISO) - if err != nil { - return err - } - - zone := s.scope.ComponentZone() - host := s.scope.ComponentHost() - overrideMap := make(map[string]infrav1.Override) - for _, o := range s.scope.ComponentOverrides() { - overrideMap[o.ID+o.Version+o.Arch] = o - } - - override := overrideMap[iso.ID()+iso.Version()+iso.Arch()] - iso.HTTPChecksum.SetHost(host) - iso.HTTPChecksum.SetPath(override.Checksum.Path) - if err := util.DownloadAndCopy(s.instanceScope, iso, zone, host, override.Path, override.URL, override.Checksum.Value, timeout); err != nil { - return err - } - return nil -} - -// MountISO mounts the ISO file to the remote instance. -func (s *Service) MountISO() error { - if !s.instanceScope.RepositoryUseISO() { - return nil - } - - mountPath := filepath.Join(file.MntDir, "repository") - dirSvc := s.getDirectoryService(mountPath, os.FileMode(filesystem.FileMode0755)) - if err := dirSvc.Make(); err != nil { - return errors.Wrapf(err, "failed to make directory %s", mountPath) - } - - iso, err := s.getISOService(s.os, s.instanceScope.Arch(), s.instanceScope.Repository().ISO) - if err != nil { - return err - } - - if _, err := s.sshClient.SudoCmd(fmt.Sprintf("sudo mount -t iso9660 -o loop %s %s", iso.RemotePath(), mountPath)); err != nil { - return errors.Wrapf(err, "mount %s at %s failed", iso.RemotePath(), mountPath) - } - s.mountPath = mountPath - return nil -} - -// UmountISO unmounts the ISO file from the remote instance. -func (s *Service) UmountISO() error { - if !s.instanceScope.RepositoryUseISO() { - return nil - } - - if _, err := s.sshClient.SudoCmd(fmt.Sprintf("sudo umount %s", s.mountPath)); err != nil { - return errors.Wrapf(err, "umount %s failed", s.mountPath) - } - return nil -} - -// UpdateAndInstall updates the linux package manager and installs some tools. -// Ex: -// apt-get update && apt-get install -y socat conntrack ipset ebtables chrony ipvsadm -// yum clean all && yum makecache && yum install -y openssl socat conntrack ipset ebtables chrony ipvsadm -func (s *Service) UpdateAndInstall() error { - if !s.instanceScope.RepositoryEnabled() { - return nil - } - - svc := s.getRepositoryService(s.os) - if svc == nil { - checkDeb, debErr := s.sshClient.SudoCmd("which apt") - if debErr == nil && strings.Contains(checkDeb, "bin") { - svc = repository.NewDeb(s.sshClient) - } - checkRpm, rpmErr := s.sshClient.SudoCmd("which yum") - if rpmErr == nil && strings.Contains(checkRpm, "bin") { - svc = repository.NewRPM(s.sshClient) - } - - if debErr != nil && rpmErr != nil { - return errors.Errorf("failed to find package manager: %v, %v", debErr, rpmErr) - } else if debErr == nil && rpmErr == nil { - return errors.New("can't detect the main package repository, only one of apt or yum is supported") - } - } - - if s.instanceScope.RepositoryUseISO() { - if err := svc.Add(s.mountPath); err != nil { - return errors.Wrapf(err, "failed to add local repository %s", s.mountPath) - } - } - - if s.instanceScope.Repository().Update { - if err := svc.Update(); err != nil { - return errors.Wrap(err, "failed to update os repository") - } - } - if err := svc.Install(s.instanceScope.Repository().Packages...); err != nil { - return errors.Wrap(err, "failed to use the repository to install software") - } - return nil -} diff --git a/pkg/service/repository/service.go b/pkg/service/repository/service.go deleted file mode 100644 index 8cd5f42d..00000000 --- a/pkg/service/repository/service.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package repository - -import ( - "os" - - "github.com/kubesphere/kubekey/v3/pkg/clients/ssh" - "github.com/kubesphere/kubekey/v3/pkg/scope" - "github.com/kubesphere/kubekey/v3/pkg/service/operation" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/directory" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/file" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/repository" - "github.com/kubesphere/kubekey/v3/util/osrelease" -) - -// Service holds a collection of interfaces. -// The interfaces are broken down like this to group functions together. -type Service struct { - sshClient ssh.Interface - scope scope.KKInstanceScope - instanceScope *scope.InstanceScope - - os *osrelease.Data - mountPath string - - repositoryFactory func(sshClient ssh.Interface, os *osrelease.Data) operation.Repository - isoFactory func(sshClient ssh.Interface, arch, isoName string) (*file.ISO, error) - directoryFactory func(sshClient ssh.Interface, path string, mode os.FileMode) operation.Directory -} - -// NewService returns a new service given the remote instance kubekey build-in repository client. -func NewService(sshClient ssh.Interface, scope scope.KKInstanceScope, instanceScope *scope.InstanceScope) *Service { - return &Service{ - sshClient: sshClient, - scope: scope, - instanceScope: instanceScope, - } -} - -func (s *Service) getRepositoryService(os *osrelease.Data) operation.Repository { - if s.repositoryFactory != nil { - return s.repositoryFactory(s.sshClient, os) - } - return repository.NewService(s.sshClient, os) -} - -func (s *Service) getISOService(os *osrelease.Data, arch string, isoName string) (*file.ISO, error) { - if s.isoFactory != nil { - return s.isoFactory(s.sshClient, arch, isoName) - } - return file.NewISO(s.sshClient, s.scope.RootFs(), os, arch, isoName) -} - -func (s *Service) getDirectoryService(path string, mode os.FileMode) operation.Directory { - if s.directoryFactory != nil { - return s.directoryFactory(s.sshClient, path, mode) - } - return directory.NewService(s.sshClient, path, mode) -} diff --git a/pkg/service/util/doc.go b/pkg/service/util/doc.go deleted file mode 100644 index 5000de40..00000000 --- a/pkg/service/util/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package util contains utility functions for the service. -package util diff --git a/pkg/service/util/downloader.go b/pkg/service/util/downloader.go deleted file mode 100644 index 145d38fa..00000000 --- a/pkg/service/util/downloader.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package util - -import ( - "time" - - "github.com/kubesphere/kubekey/v3/pkg/scope" - "github.com/kubesphere/kubekey/v3/pkg/service/operation" - "github.com/kubesphere/kubekey/v3/pkg/service/operation/file/checksum" -) - -// DownloadAndCopy downloads and copies files to the remote instance. -func DownloadAndCopy(instanceScope *scope.InstanceScope, b operation.Binary, zone, host, path, url, checksumStr string, timeout time.Duration) error { - if b.RemoteExist() { - return nil - } - - b.AppendChecksum(checksum.NewStringChecksum(checksumStr)) - if !(b.LocalExist() && b.CompareChecksum() == nil) { - // Only the host is an empty string, we can set up the zone. - // Because the URL path which in the QingStor is not the same as the default. - if host == "" { - b.SetZone(zone) - } - - // Always try to set the "host, path, url, checksum". - // If the these vars are empty strings, it will not make any changes. - b.SetHost(host) - b.SetPath(path) - b.SetURL(url) - - instanceScope.V(4).Info("download binary", "binary", b.Name(), "version", b.Version(), - "url", b.URL().String()) - if err := b.Get(timeout); err != nil { - return err - } - if err := b.CompareChecksum(); err != nil { - return err - } - } - - if err := b.Copy(true); err != nil { - return err - } - return nil -} diff --git a/pkg/util/filesystem/common.go b/pkg/util/filesystem/common.go deleted file mode 100644 index ff449eaa..00000000 --- a/pkg/util/filesystem/common.go +++ /dev/null @@ -1,33 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package filesystem - -const ( - // DefaultLocalTmpDir represents the CAPKK default local tmp directory - DefaultLocalTmpDir = "/var/lib/kubekey" -) - -const ( - // FileMode0777 represents the file mode 0755 - FileMode0777 = 0777 - // FileMode0755 represents the file mode 0755 - FileMode0755 = 0755 - // FileMode0644 represents the file mode 0644 - FileMode0644 = 0644 - // FileMode0664 represents the file mode 0664 - FileMode0664 = 0664 -) diff --git a/pkg/util/filesystem/doc.go b/pkg/util/filesystem/doc.go deleted file mode 100644 index a5647921..00000000 --- a/pkg/util/filesystem/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package filesystem implements the operation of the local file system. -package filesystem diff --git a/pkg/util/filesystem/filemode.go b/pkg/util/filesystem/filemode.go deleted file mode 100644 index e7d897e1..00000000 --- a/pkg/util/filesystem/filemode.go +++ /dev/null @@ -1,50 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package filesystem - -import ( - "os" - "syscall" -) - -const ( - sIsuid = syscall.S_ISUID - sIsgid = syscall.S_ISGID - sIsvtx = syscall.S_ISVTX -) - -// ToChmodPerm converts Go permission bits to POSIX permission bits. -// -// This differs from fromFileMode in that we preserve the POSIX versions of -// setuid, setgid and sticky in m, because we've historically supported those -// bits, and we mask off any non-permission bits. -func ToChmodPerm(m os.FileMode) (perm uint32) { - const mask = os.ModePerm | sIsuid | sIsgid | sIsvtx - perm = uint32(m & mask) - - if m&os.ModeSetuid != 0 { - perm |= sIsuid - } - if m&os.ModeSetgid != 0 { - perm |= sIsgid - } - if m&os.ModeSticky != 0 { - perm |= sIsvtx - } - - return perm -} diff --git a/pkg/util/filesystem/filemode_test.go b/pkg/util/filesystem/filemode_test.go deleted file mode 100644 index dd787e33..00000000 --- a/pkg/util/filesystem/filemode_test.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package filesystem - -import ( - "os" - "testing" -) - -func TestToChmodPerm(t *testing.T) { - tests := []struct { - mode os.FileMode - wantPerm uint32 - }{ - { - os.FileMode(0000), - 0000, - }, - { - os.FileMode(0777), - 0777, - }, - { - os.FileMode(0660), - 0660, - }, - { - os.FileMode(0755), - 0755, - }, - } - for _, tt := range tests { - t.Run("", func(t *testing.T) { - if gotPerm := ToChmodPerm(tt.mode); gotPerm != tt.wantPerm { - t.Errorf("ToChmodPerm() = %v, want %v", gotPerm, tt.wantPerm) - } - }) - } -} diff --git a/pkg/util/filesystem/fs.go b/pkg/util/filesystem/fs.go deleted file mode 100644 index eac8adba..00000000 --- a/pkg/util/filesystem/fs.go +++ /dev/null @@ -1,91 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package filesystem - -import ( - "fmt" - "os" - - "github.com/kubesphere/kubekey/v3/pkg/util/hash" -) - -// FileSystem is a filesystem implementation -type FileSystem struct { -} - -// NewFileSystem returns a new CAPKK local filesystem implementation -func NewFileSystem() Interface { - return FileSystem{} -} - -// Stat returns the FileInfo for the given path -func (f FileSystem) Stat(name string) (os.FileInfo, error) { - return os.Stat(name) -} - -// MkdirAll the same as os.MkdirAll(). -func (f FileSystem) MkdirAll(path string) error { - return os.MkdirAll(path, os.ModePerm) -} - -// MD5Sum returns the file MD5 sum for the given local path. -func (f FileSystem) MD5Sum(localPath string) string { - md5, err := hash.FileMD5(localPath) - if err != nil { - return "" - } - return md5 -} - -// SHA256Sum returns the file SHA256 sum for the given local path. -func (f FileSystem) SHA256Sum(localPath string) string { - sha256, err := hash.FileSHA256(localPath) - if err != nil { - return "" - } - return sha256 -} - -// MkLocalTmpDir creates a temporary directory and returns the path -func (f FileSystem) MkLocalTmpDir() (string, error) { - tempDir, err := os.MkdirTemp(DefaultLocalTmpDir, ".Tmp-") - if err != nil { - return "", err - } - return tempDir, os.MkdirAll(tempDir, os.ModePerm) -} - -// MkLocalTmpFile creates a temporary file and returns the path. -func (f FileSystem) MkLocalTmpFile(dir, pattern string) (string, error) { - file, err := os.CreateTemp(dir, pattern) - if err != nil { - return "", err - } - _ = file.Close() - return file.Name(), nil -} - -// RemoveAll the same as os.RemoveAll(). -func (f FileSystem) RemoveAll(path ...string) error { - for _, fi := range path { - err := os.RemoveAll(fi) - if err != nil { - return fmt.Errorf("failed to remove file %s, %v", fi, err) - } - } - return nil -} diff --git a/pkg/util/filesystem/interface.go b/pkg/util/filesystem/interface.go deleted file mode 100644 index a164ab0b..00000000 --- a/pkg/util/filesystem/interface.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package filesystem - -import ( - "os" -) - -// Interface is an interface for filesystem operations -type Interface interface { - // Stat returns the FileInfo structure describing the named file. - Stat(name string) (os.FileInfo, error) - // MkdirAll the same as os.MkdirAll(). - MkdirAll(path string) error - // MD5Sum returns the file MD5 sum for the given local path. - MD5Sum(localPath string) string - // SHA256Sum returns the file SHA256 sum for the given local path. - SHA256Sum(localPath string) string - // MkLocalTmpDir creates a temporary directory and returns the path. - MkLocalTmpDir() (string, error) - // MkLocalTmpFile creates a temporary file and returns the path. - MkLocalTmpFile(dir, pattern string) (string, error) - // RemoveAll the same as os.RemoveAll(). - RemoveAll(path ...string) error -} diff --git a/pkg/util/hash/hash.go b/pkg/util/hash/hash.go deleted file mode 100644 index e792ce3a..00000000 --- a/pkg/util/hash/hash.go +++ /dev/null @@ -1,60 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package hash implements hash utilities -package hash - -import ( - "crypto/md5" // #nosec - "crypto/sha256" - "fmt" - "io" - "os" - "path/filepath" -) - -// FileMD5 count file md5 -func FileMD5(path string) (string, error) { - file, err := os.Open(filepath.Clean(path)) - if err != nil { - return "", err - } - defer file.Close() - - m := md5.New() // #nosec - if _, err := io.Copy(m, file); err != nil { - return "", err - } - - fileMd5 := fmt.Sprintf("%x", m.Sum(nil)) - return fileMd5, nil -} - -// FileSHA256 count file sha256 -func FileSHA256(path string) (string, error) { - file, err := os.Open(filepath.Clean(path)) - if err != nil { - return "", err - } - defer file.Close() - - s := sha256.New() - if _, err := io.Copy(s, file); err != nil { - return "", err - } - - return fmt.Sprintf("%x", s.Sum(nil)), nil -} diff --git a/pkg/util/util.go b/pkg/util/util.go deleted file mode 100644 index 42b671ca..00000000 --- a/pkg/util/util.go +++ /dev/null @@ -1,32 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package util implements utilities. -package util - -// ArchAlias returns the alias of cpu's architecture. -// amd64: x86_64 -// arm64: aarch64 -func ArchAlias(arch string) string { - switch arch { - case "amd64": - return "x86_64" - case "arm64": - return "aarch64" - default: - return "" - } -} diff --git a/scripts/ci-lint-dockerfiles.sh b/scripts/ci-lint-dockerfiles.sh deleted file mode 100755 index c62fb3bd..00000000 --- a/scripts/ci-lint-dockerfiles.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Copyright 2022 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -HADOLINT_VER=${1:-latest} -HADOLINT_FAILURE_THRESHOLD=${2:-warning} - -FILES=$(find -- * -name Dockerfile) -while read -r file; do - echo "Linting: ${file}" - # Configure the linter to fail for warnings and errors. Can be set to: error | warning | info | style | ignore | none - docker run --rm -i ghcr.io/hadolint/hadolint:"${HADOLINT_VER}" hadolint --failure-threshold "${HADOLINT_FAILURE_THRESHOLD}" - < "${file}" -done <<< "${FILES}" diff --git a/scripts/docker-install.sh b/scripts/docker-install.sh deleted file mode 100755 index 9bb70410..00000000 --- a/scripts/docker-install.sh +++ /dev/null @@ -1,526 +0,0 @@ -#!/bin/sh -set -e -# Docker CE for Linux installation script -# -# See https://docs.docker.com/engine/install/ for the installation steps. -# -# This script is meant for quick & easy install via: -# $ curl -fsSL https://get.docker.com -o get-docker.sh -# $ sh get-docker.sh -# -# For test builds (ie. release candidates): -# $ curl -fsSL https://test.docker.com -o test-docker.sh -# $ sh test-docker.sh -# -# NOTE: Make sure to verify the contents of the script -# you downloaded matches the contents of install.sh -# located at https://github.com/docker/docker-install -# before executing. -# -# Git commit from https://github.com/docker/docker-install when -# the script was uploaded (Should only be modified by upload job): -SCRIPT_COMMIT_SHA="7cae5f8b0decc17d6571f9f52eb840fbc13b2737" - - -# The channel to install from: -# * nightly -# * test -# * stable -# * edge (deprecated) -DEFAULT_CHANNEL_VALUE="stable" -if [ -z "$CHANNEL" ]; then - CHANNEL=$DEFAULT_CHANNEL_VALUE -fi - -#DEFAULT_DOWNLOAD_URL="https://download.docker.com" -DEFAULT_DOWNLOAD_URL="https://mirrors.aliyun.com/docker-ce" -if [ -z "$DOWNLOAD_URL" ]; then - DOWNLOAD_URL=$DEFAULT_DOWNLOAD_URL -fi - -DEFAULT_REPO_FILE="docker-ce.repo" -if [ -z "$REPO_FILE" ]; then - REPO_FILE="$DEFAULT_REPO_FILE" -fi - -mirror='' -DRY_RUN=${DRY_RUN:-} -while [ $# -gt 0 ]; do - case "$1" in - --mirror) - mirror="$2" - shift - ;; - --dry-run) - DRY_RUN=1 - ;; - --*) - echo "Illegal option $1" - ;; - esac - shift $(( $# > 0 ? 1 : 0 )) -done - -case "$mirror" in - Aliyun) - DOWNLOAD_URL="https://mirrors.aliyun.com/docker-ce" - ;; - AzureChinaCloud) - DOWNLOAD_URL="https://mirror.azure.cn/docker-ce" - ;; -esac - -# docker-ce-rootless-extras is packaged since Docker 20.10.0 -has_rootless_extras="1" -if echo "$VERSION" | grep -q '^1'; then - has_rootless_extras= -fi - -command_exists() { - command -v "$@" > /dev/null 2>&1 -} - -is_dry_run() { - if [ -z "$DRY_RUN" ]; then - return 1 - else - return 0 - fi -} - -is_wsl() { - case "$(uname -r)" in - *microsoft* ) true ;; # WSL 2 - *Microsoft* ) true ;; # WSL 1 - * ) false;; - esac -} - -is_darwin() { - case "$(uname -s)" in - *darwin* ) true ;; - *Darwin* ) true ;; - * ) false;; - esac -} - -deprecation_notice() { - distro=$1 - date=$2 - echo - echo "DEPRECATION WARNING:" - echo " The distribution, $distro, will no longer be supported in this script as of $date." - echo " If you feel this is a mistake please submit an issue at https://github.com/docker/docker-install/issues/new" - echo - sleep 10 -} - -get_distribution() { - lsb_dist="" - # Every system that we officially support has /etc/os-release - if [ -r /etc/os-release ]; then - lsb_dist="$(. /etc/os-release && echo "$ID")" - fi - # Returning an empty string here should be alright since the - # case statements don't act unless you provide an actual value - echo "$lsb_dist" -} - -add_debian_backport_repo() { - debian_version="$1" - backports="deb http://ftp.debian.org/debian $debian_version-backports main" - if ! grep -Fxq "$backports" /etc/apt/sources.list; then - (set -x; $sh_c "echo \"$backports\" >> /etc/apt/sources.list") - fi -} - -echo_docker_as_nonroot() { - if is_dry_run; then - return - fi - if command_exists docker && [ -e /var/run/docker.sock ]; then - ( - set -x - $sh_c 'docker version' - ) || true - fi - - # intentionally mixed spaces and tabs here -- tabs are stripped by "<<-EOF", spaces are kept in the output - echo - echo "================================================================================" - echo - if [ -n "$has_rootless_extras" ]; then - echo "To run Docker as a non-privileged user, consider setting up the" - echo "Docker daemon in rootless mode for your user:" - echo - echo " dockerd-rootless-setuptool.sh install" - echo - echo "Visit https://docs.docker.com/go/rootless/ to learn about rootless mode." - echo - fi - echo - echo "To run the Docker daemon as a fully privileged service, but granting non-root" - echo "users access, refer to https://docs.docker.com/go/daemon-access/" - echo - echo "WARNING: Access to the remote API on a privileged Docker daemon is equivalent" - echo " to root access on the host. Refer to the 'Docker daemon attack surface'" - echo " documentation for details: https://docs.docker.com/go/attack-surface/" - echo - echo "================================================================================" - echo -} - -# Check if this is a forked Linux distro -check_forked() { - - # Check for lsb_release command existence, it usually exists in forked distros - if command_exists lsb_release; then - # Check if the `-u` option is supported - set +e - lsb_release -a -u > /dev/null 2>&1 - lsb_release_exit_code=$? - set -e - - # Check if the command has exited successfully, it means we're in a forked distro - if [ "$lsb_release_exit_code" = "0" ]; then - # Print info about current distro - cat <<-EOF - You're using '$lsb_dist' version '$dist_version'. - EOF - - # Get the upstream release info - lsb_dist=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'id' | cut -d ':' -f 2 | tr -d '[:space:]') - dist_version=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'codename' | cut -d ':' -f 2 | tr -d '[:space:]') - - # Print info about upstream distro - cat <<-EOF - Upstream release is '$lsb_dist' version '$dist_version'. - EOF - else - if [ -r /etc/debian_version ] && [ "$lsb_dist" != "ubuntu" ] && [ "$lsb_dist" != "raspbian" ]; then - if [ "$lsb_dist" = "osmc" ]; then - # OSMC runs Raspbian - lsb_dist=raspbian - else - # We're Debian and don't even know it! - lsb_dist=debian - fi - dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')" - case "$dist_version" in - 10) - dist_version="buster" - ;; - 9) - dist_version="stretch" - ;; - 8|'Kali Linux 2') - dist_version="jessie" - ;; - esac - fi - fi - fi -} - -semverParse() { - major="${1%%.*}" - minor="${1#$major.}" - minor="${minor%%.*}" - patch="${1#$major.$minor.}" - patch="${patch%%[-.]*}" -} - -do_install() { - echo "# Executing docker install script, commit: $SCRIPT_COMMIT_SHA" - - if command_exists docker; then - docker_version="$(docker -v | cut -d ' ' -f3 | cut -d ',' -f1)" - MAJOR_W=1 - MINOR_W=10 - - semverParse "$docker_version" - - shouldWarn=0 - if [ "$major" -lt "$MAJOR_W" ]; then - shouldWarn=1 - fi - - if [ "$major" -le "$MAJOR_W" ] && [ "$minor" -lt "$MINOR_W" ]; then - shouldWarn=1 - fi - - cat >&2 <<-'EOF' - Warning: the "docker" command appears to already exist on this system. - - If you already have Docker installed, this script can cause trouble, which is - why we're displaying this warning and provide the opportunity to cancel the - installation. - - If you installed the current Docker package using this script and are using it - EOF - - if [ $shouldWarn -eq 1 ]; then - cat >&2 <<-'EOF' - again to update Docker, we urge you to migrate your image store before upgrading - to v1.10+. - - You can find instructions for this here: - https://github.com/docker/docker/wiki/Engine-v1.10.0-content-addressability-migration - EOF - else - cat >&2 <<-'EOF' - again to update Docker, you can safely ignore this message. - EOF - fi - - cat >&2 <<-'EOF' - - You may press Ctrl+C now to abort this script. - EOF - ( set -x; sleep 20 ) - fi - - user="$(id -un 2>/dev/null || true)" - - sh_c='sh -c' - if [ "$user" != 'root' ]; then - if command_exists sudo; then - sh_c='sudo -E sh -c' - elif command_exists su; then - sh_c='su -c' - else - cat >&2 <<-'EOF' - Error: this installer needs the ability to run commands as root. - We are unable to find either "sudo" or "su" available to make this happen. - EOF - exit 1 - fi - fi - - if is_dry_run; then - sh_c="echo" - fi - - # perform some very rudimentary platform detection - lsb_dist=$( get_distribution ) - lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')" - - if is_wsl; then - echo - echo "WSL DETECTED: We recommend using Docker Desktop for Windows." - echo "Please get Docker Desktop from https://www.docker.com/products/docker-desktop" - echo - cat >&2 <<-'EOF' - - You may press Ctrl+C now to abort this script. - EOF - ( set -x; sleep 20 ) - fi - - case "$lsb_dist" in - - ubuntu) - if command_exists lsb_release; then - dist_version="$(lsb_release --codename | cut -f2)" - fi - if [ -z "$dist_version" ] && [ -r /etc/lsb-release ]; then - dist_version="$(. /etc/lsb-release && echo "$DISTRIB_CODENAME")" - fi - ;; - - debian|raspbian) - dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')" - case "$dist_version" in - 10) - dist_version="buster" - ;; - 9) - dist_version="stretch" - ;; - 8) - dist_version="jessie" - ;; - esac - ;; - - centos|rhel) - if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then - dist_version="$(. /etc/os-release && echo "$VERSION_ID")" - fi - ;; - - *) - if command_exists lsb_release; then - dist_version="$(lsb_release --release | cut -f2)" - fi - if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then - dist_version="$(. /etc/os-release && echo "$VERSION_ID")" - fi - ;; - - esac - - # Check if this is a forked Linux distro - check_forked - - # Run setup for each distro accordingly - case "$lsb_dist" in - ubuntu|debian|raspbian) - pre_reqs="apt-transport-https ca-certificates curl" - if [ "$lsb_dist" = "debian" ]; then - # libseccomp2 does not exist for debian jessie main repos for aarch64 - if [ "$(uname -m)" = "aarch64" ] && [ "$dist_version" = "jessie" ]; then - add_debian_backport_repo "$dist_version" - fi - fi - - if ! command -v gpg > /dev/null; then - pre_reqs="$pre_reqs gnupg" - fi - apt_repo="deb [arch=$(dpkg --print-architecture)] $DOWNLOAD_URL/linux/$lsb_dist $dist_version $CHANNEL" - ( - if ! is_dry_run; then - set -x - fi - $sh_c 'apt-get update -qq >/dev/null' - $sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq $pre_reqs >/dev/null" - $sh_c "curl -fsSL \"$DOWNLOAD_URL/linux/$lsb_dist/gpg\" | apt-key add -qq - >/dev/null" - $sh_c "echo \"$apt_repo\" > /etc/apt/sources.list.d/docker.list" - $sh_c 'apt-get update -qq >/dev/null' - ) - pkg_version="" - if [ -n "$VERSION" ]; then - if is_dry_run; then - echo "# WARNING: VERSION pinning is not supported in DRY_RUN" - else - # Will work for incomplete versions IE (17.12), but may not actually grab the "latest" if in the test channel - pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/~ce~.*/g" | sed "s/-/.*/g").*-0~$lsb_dist" - search_command="apt-cache madison 'docker-ce' | grep '$pkg_pattern' | head -1 | awk '{\$1=\$1};1' | cut -d' ' -f 3" - pkg_version="$($sh_c "$search_command")" - echo "INFO: Searching repository for VERSION '$VERSION'" - echo "INFO: $search_command" - if [ -z "$pkg_version" ]; then - echo - echo "ERROR: '$VERSION' not found amongst apt-cache madison results" - echo - exit 1 - fi - search_command="apt-cache madison 'docker-ce-cli' | grep '$pkg_pattern' | head -1 | awk '{\$1=\$1};1' | cut -d' ' -f 3" - # Don't insert an = for cli_pkg_version, we'll just include it later - cli_pkg_version="$($sh_c "$search_command")" - pkg_version="=$pkg_version" - fi - fi - ( - if ! is_dry_run; then - set -x - fi - if [ -n "$cli_pkg_version" ]; then - $sh_c "apt-get install -y -qq --no-install-recommends docker-ce-cli=$cli_pkg_version >/dev/null" - fi - $sh_c "apt-get install -y -qq --no-install-recommends docker-ce$pkg_version >/dev/null" - # shellcheck disable=SC2030 - if [ -n "$has_rootless_extras" ]; then - # Install docker-ce-rootless-extras without "--no-install-recommends", so as to install slirp4netns when available - $sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq docker-ce-rootless-extras$pkg_version >/dev/null" - fi - ) - echo_docker_as_nonroot - exit 0 - ;; - centos|fedora|rhel) - yum_repo="$DOWNLOAD_URL/linux/$lsb_dist/$REPO_FILE" - if ! curl -Ifs "$yum_repo" > /dev/null; then - echo "Error: Unable to curl repository file $yum_repo, is it valid?" - exit 1 - fi - if [ "$lsb_dist" = "fedora" ]; then - pkg_manager="dnf" - config_manager="dnf config-manager" - enable_channel_flag="--set-enabled" - disable_channel_flag="--set-disabled" - pre_reqs="dnf-plugins-core" - pkg_suffix="fc$dist_version" - else - pkg_manager="yum" - config_manager="yum-config-manager" - enable_channel_flag="--enable" - disable_channel_flag="--disable" - pre_reqs="yum-utils" - pkg_suffix="el" - fi - ( - if ! is_dry_run; then - set -x - fi - $sh_c "$pkg_manager install -y -q $pre_reqs" - $sh_c "$config_manager --add-repo $yum_repo" - - if [ "$CHANNEL" != "stable" ]; then - $sh_c "$config_manager $disable_channel_flag docker-ce-*" - $sh_c "$config_manager $enable_channel_flag docker-ce-$CHANNEL" - fi - $sh_c "$pkg_manager makecache" - ) - pkg_version="" - if [ -n "$VERSION" ]; then - if is_dry_run; then - echo "# WARNING: VERSION pinning is not supported in DRY_RUN" - else - pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/\\\\.ce.*/g" | sed "s/-/.*/g").*$pkg_suffix" - search_command="$pkg_manager list --showduplicates 'docker-ce' | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'" - pkg_version="$($sh_c "$search_command")" - echo "INFO: Searching repository for VERSION '$VERSION'" - echo "INFO: $search_command" - if [ -z "$pkg_version" ]; then - echo - echo "ERROR: '$VERSION' not found amongst $pkg_manager list results" - echo - exit 1 - fi - search_command="$pkg_manager list --showduplicates 'docker-ce-cli' | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'" - # It's okay for cli_pkg_version to be blank, since older versions don't support a cli package - cli_pkg_version="$($sh_c "$search_command" | cut -d':' -f 2)" - # Cut out the epoch and prefix with a '-' - pkg_version="-$(echo "$pkg_version" | cut -d':' -f 2)" - fi - fi - ( - if ! is_dry_run; then - set -x - fi - # install the correct cli version first - if [ -n "$cli_pkg_version" ]; then - $sh_c "$pkg_manager install -y -q docker-ce-cli-$cli_pkg_version" - fi - $sh_c "$pkg_manager install -y -q docker-ce$pkg_version" - # shellcheck disable=SC2031 - if [ -n "$has_rootless_extras" ]; then - $sh_c "$pkg_manager install -y -q docker-ce-rootless-extras$pkg_version" - fi - ) - exit 0 - ;; - *) - if [ -z "$lsb_dist" ]; then - if is_darwin; then - echo - echo "ERROR: Unsupported operating system 'macOS'" - echo "Please get Docker Desktop from https://www.docker.com/products/docker-desktop" - echo - exit 1 - fi - fi - echo - echo "ERROR: Unsupported distribution '$lsb_dist'" - echo - exit 1 - ;; - esac - - exit 1 -} - -# wrapped up in a function so that we have some protection against only getting -# half the file during "curl | sh" -do_install diff --git a/scripts/downloadKubekey.sh b/scripts/downloadKubekey.sh deleted file mode 100755 index 8fd9b947..00000000 --- a/scripts/downloadKubekey.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/sh - -# Copyright 2020 The KubeSphere Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ISLINUX=true -OSTYPE="linux" - -if [ "x$(uname)" != "xLinux" ]; then - echo "" - echo 'Warning: Non-Linux operating systems are not supported! After downloading, please copy the tar.gz file to linux.' - ISLINUX=false -fi - -# Fetch latest version -if [ "x${VERSION}" = "x" ]; then - VERSION="$(curl -sL https://api.github.com/repos/kubesphere/kubekey/releases | - grep -o 'download/v[0-9]*.[0-9]*.[0-9]*/' | - sort --version-sort | - tail -1 | awk -F'/' '{ print $2}')" - VERSION="${VERSION##*/}" -fi - -if [ -z "${ARCH}" ]; then - case "$(uname -m)" in - x86_64) - ARCH=amd64 - ;; - armv8*) - ARCH=arm64 - ;; - aarch64*) - ARCH=arm64 - ;; - *) - echo "${ARCH}, isn't supported" - exit 1 - ;; - esac -fi - -if [ "x${VERSION}" = "x" ]; then - echo "Unable to get latest Kubekey version. Set VERSION env var and re-run. For example: export VERSION=v1.0.0" - echo "" - exit -fi - -DOWNLOAD_URL="https://github.com/kubesphere/kubekey/releases/download/${VERSION}/kubekey-${VERSION}-${OSTYPE}-${ARCH}.tar.gz" -if [ "x${KKZONE}" = "xcn" ]; then - DOWNLOAD_URL="https://kubernetes.pek3b.qingstor.com/kubekey/releases/download/${VERSION}/kubekey-${VERSION}-${OSTYPE}-${ARCH}.tar.gz" -fi - -echo "" -echo "Downloading kubekey ${VERSION} from ${DOWNLOAD_URL} ..." -echo "" - -curl -fsLO "$DOWNLOAD_URL" -if [ $? -ne 0 ]; then - echo "" - echo "Failed to download Kubekey ${VERSION} !" - echo "" - echo "Please verify the version you are trying to download." - echo "" - exit -fi - -if [ ${ISLINUX} = true ]; then - filename="kubekey-${VERSION}-${OSTYPE}-${ARCH}.tar.gz" - ret='0' - command -v tar >/dev/null 2>&1 || { ret='1'; } - if [ "$ret" -eq 0 ]; then - tar -xzf "${filename}" - else - echo "Kubekey ${VERSION} Download Complete!" - echo "" - echo "Try to unpack the ${filename} failed." - echo "tar: command not found, please unpack the ${filename} manually." - exit - fi -fi - -echo "" -echo "Kubekey ${VERSION} Download Complete!" -echo "" - diff --git a/scripts/go_install.sh b/scripts/go_install.sh deleted file mode 100755 index a07b8e0f..00000000 --- a/scripts/go_install.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2021 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -if [ -z "${1}" ]; then - echo "must provide module as first parameter" - exit 1 -fi - -if [ -z "${2}" ]; then - echo "must provide binary name as second parameter" - exit 1 -fi - -if [ -z "${3}" ]; then - echo "must provide version as third parameter" - exit 1 -fi - -if [ -z "${GOBIN}" ]; then - echo "GOBIN is not set. Must set GOBIN to install the bin in a specified directory." - exit 1 -fi - -rm -f "${GOBIN}/${2}"* || true - -# install the golang module specified as the first argument -go install "${1}@${3}" -mv "${GOBIN}/${2}" "${GOBIN}/${2}-${3}" -ln -sf "${GOBIN}/${2}-${3}" "${GOBIN}/${2}" diff --git a/scripts/harborCreateRegistriesAndReplications.sh b/scripts/harborCreateRegistriesAndReplications.sh deleted file mode 100644 index 86375c48..00000000 --- a/scripts/harborCreateRegistriesAndReplications.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - - -function createRegistries() { - - # create registry - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master1_Address}/api/v2.0/registries" -d "{\"name\": \"master1_2_master2\", \"type\": \"harbor\", \"url\":\"https://${master2_Address}:7443\", \"credential\": {\"access_key\": \"${Harbor_User}\", \"access_secret\": \"${Harbor_Passwd}\"}, \"insecure\": true}" - # create registry - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master1_Address}/api/v2.0/registries" -d "{\"name\": \"master1_2_master3\", \"type\": \"harbor\", \"url\":\"https://${master3_Address}:7443\", \"credential\": {\"access_key\": \"${Harbor_User}\", \"access_secret\": \"${Harbor_Passwd}\"}, \"insecure\": true}" - - # create registry - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master2_Address}/api/v2.0/registries" -d "{\"name\": \"master2_2_master1\", \"type\": \"harbor\", \"url\":\"https://${master1_Address}:7443\", \"credential\": {\"access_key\": \"${Harbor_User}\", \"access_secret\": \"${Harbor_Passwd}\"}, \"insecure\": true}" - # create registry - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master2_Address}/api/v2.0/registries" -d "{\"name\": \"master2_2_master3\", \"type\": \"harbor\", \"url\":\"https://${master3_Address}:7443\", \"credential\": {\"access_key\": \"${Harbor_User}\", \"access_secret\": \"${Harbor_Passwd}\"}, \"insecure\": true}" - - # create registry - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master3_Address}/api/v2.0/registries" -d "{\"name\": \"master3_2_master1\", \"type\": \"harbor\", \"url\":\"https://${master1_Address}:7443\", \"credential\": {\"access_key\": \"${Harbor_User}\", \"access_secret\": \"${Harbor_Passwd}\"}, \"insecure\": true}" - # create registry - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master3_Address}/api/v2.0/registries" -d "{\"name\": \"master3_2_master2\", \"type\": \"harbor\", \"url\":\"https://${master2_Address}:7443\", \"credential\": {\"access_key\": \"${Harbor_User}\", \"access_secret\": \"${Harbor_Passwd}\"}, \"insecure\": true}" - -} - -function listRegistries() { - curl -k -u $Harbor_UserPwd -X GET -H "Content-Type: application/json" "https://${Harbor_master1_Address}/api/v2.0/registries" - curl -k -u $Harbor_UserPwd -X GET -H "Content-Type: application/json" "https://${Harbor_master2_Address}/api/v2.0/registries" - curl -k -u $Harbor_UserPwd -X GET -H "Content-Type: application/json" "https://${Harbor_master3_Address}/api/v2.0/registries" - -} - -function createReplication() { - - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master1_Address}/api/v2.0/replication/policies" -d "{\"name\": \"master1_2_master2\", \"enabled\": true, \"deletion\":true, \"override\":true, \"replicate_deletion\":true, \"dest_registry\":{ \"id\": 1, \"name\": \"master1_2_master2\"}, \"trigger\": {\"type\": \"event_based\"}, \"dest_namespace_replace_count\":1 }" - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master1_Address}/api/v2.0/replication/policies" -d "{\"name\": \"master1_2_master3\", \"enabled\": true, \"deletion\":true, \"override\":true, \"replicate_deletion\":true, \"dest_registry\":{ \"id\": 2, \"name\": \"master1_2_master3\"}, \"trigger\": {\"type\": \"event_based\"}, \"dest_namespace_replace_count\":1 }" - - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master2_Address}/api/v2.0/replication/policies" -d "{\"name\": \"master2_2_master1\", \"enabled\": true, \"deletion\":true, \"override\":true, \"replicate_deletion\":true, \"dest_registry\":{ \"id\": 1, \"name\": \"master2_2_master1\"}, \"trigger\": {\"type\": \"event_based\"}, \"dest_namespace_replace_count\":1 }" - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master2_Address}/api/v2.0/replication/policies" -d "{\"name\": \"master2_2_master3\", \"enabled\": true, \"deletion\":true, \"override\":true, \"replicate_deletion\":true, \"dest_registry\":{ \"id\": 2, \"name\": \"master2_2_master3\"}, \"trigger\": {\"type\": \"event_based\"}, \"dest_namespace_replace_count\":1 }" - - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master3_Address}/api/v2.0/replication/policies" -d "{\"name\": \"master3_2_master1\", \"enabled\": true, \"deletion\":true, \"override\":true, \"replicate_deletion\":true, \"dest_registry\":{ \"id\": 1, \"name\": \"master3_2_master1\"}, \"trigger\": {\"type\": \"event_based\"}, \"dest_namespace_replace_count\":1 }" - curl -k -u $Harbor_UserPwd -X POST -H "Content-Type: application/json" "https://${Harbor_master3_Address}/api/v2.0/replication/policies" -d "{\"name\": \"master3_2_master2\", \"enabled\": true, \"deletion\":true, \"override\":true, \"replicate_deletion\":true, \"dest_registry\":{ \"id\": 2, \"name\": \"master3_2_master2\"}, \"trigger\": {\"type\": \"event_based\"}, \"dest_namespace_replace_count\":1 }" -} - -function listReplications() { - - curl -k -u $Harbor_UserPwd -X GET -H "Content-Type: application/json" "https://${Harbor_master1_Address}/api/v2.0/replication/policies" - curl -k -u $Harbor_UserPwd -X GET -H "Content-Type: application/json" "https://${Harbor_master2_Address}/api/v2.0/replication/policies" - curl -k -u $Harbor_UserPwd -X GET -H "Content-Type: application/json" "https://${Harbor_master3_Address}/api/v2.0/replication/policies" -} - -#### main ###### -Harbor_master1_Address=master1:7443 -master1_Address=192.168.122.61 -Harbor_master2_Address=master2:7443 -master2_Address=192.168.122.62 -Harbor_master3_Address=master3:7443 -master3_Address=192.168.122.63 -Harbor_User=admin #登录Harbor的用户 -Harbor_Passwd="Harbor12345" #登录Harbor的用户密码 -Harbor_UserPwd="$Harbor_User:$Harbor_Passwd" - - -createRegistries -listRegistries -createReplication -listReplications diff --git a/scripts/harbor_keepalived/check_harbor.sh b/scripts/harbor_keepalived/check_harbor.sh deleted file mode 100644 index 0a3fb0bd..00000000 --- a/scripts/harbor_keepalived/check_harbor.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -#count=$(docker-compose -f /opt/harbor/docker-compose.yml ps -a|grep healthy|wc -l) -# 不能频繁调用docker-compose 否则会有非常多的临时目录被创建：/tmp/_MEI* -count=$(docker ps |grep goharbor|grep healthy|wc -l) -status=$(ss -tlnp|grep -w 443|wc -l) -if [ $count -ne 11 -a ];then - exit 8 -elif [ $status -lt 2 ];then - exit 9 -else - exit 0 -fi diff --git a/scripts/harbor_keepalived/docker-compose-keepalived-backup.yaml b/scripts/harbor_keepalived/docker-compose-keepalived-backup.yaml deleted file mode 100644 index 7328f96a..00000000 --- a/scripts/harbor_keepalived/docker-compose-keepalived-backup.yaml +++ /dev/null @@ -1,14 +0,0 @@ -version: '3.8' - -# Docker-Compose 单容器使用参考 YAML 配置文件 -# 更多配置参数请参考镜像 README.md 文档中说明 -services: - keepalived: - image: 'registry.cn-shenzhen.aliyuncs.com/colovu/keepalived:2.1' - privileged: true - network_mode: host - volumes: - - ./keepalived-backup.conf:/srv/conf/keepalived/keepalived.conf - - ./check_harbor.sh:/srv/conf/keepalived/check_harbor.sh - container_name: keepalived - restart: on-failure diff --git a/scripts/harbor_keepalived/docker-compose-keepalived-master.yaml b/scripts/harbor_keepalived/docker-compose-keepalived-master.yaml deleted file mode 100644 index 64f35aee..00000000 --- a/scripts/harbor_keepalived/docker-compose-keepalived-master.yaml +++ /dev/null @@ -1,14 +0,0 @@ -version: '3.8' - -# Docker-Compose 单容器使用参考 YAML 配置文件 -# 更多配置参数请参考镜像 README.md 文档中说明 -services: - keepalived: - image: 'registry.cn-shenzhen.aliyuncs.com/colovu/keepalived:2.1' - privileged: true - network_mode: host - volumes: - - ./keepalived-master.conf:/srv/conf/keepalived/keepalived.conf - - ./check_harbor.sh:/srv/conf/keepalived/check_harbor.sh - container_name: keepalived - restart: on-failure diff --git a/scripts/harbor_keepalived/keepalived-backup.conf b/scripts/harbor_keepalived/keepalived-backup.conf deleted file mode 100644 index be916c90..00000000 --- a/scripts/harbor_keepalived/keepalived-backup.conf +++ /dev/null @@ -1,31 +0,0 @@ -vrrp_script check_harbor { - script "/srv/conf/keepalived/check_harbor.sh" - interval 10 # 间隔时间，单位为秒，默认1秒 - fall 2 # 脚本几次失败转换为失败 - rise 2 # 脚本连续监测成功后，把服务器从失败标记为成功的次数 - timeout 5 - init_fail -} -global_defs { - script_user root - router_id harbor-ha - enable_script_security - lvs_sync_daemon ens3 VI_1 -} -vrrp_instance VI_1 { - state BACKUP - interface ens3 - virtual_router_id 31 # 如果同一个局域网中有多套keepalive，那么要保证该id唯一 - priority 50 - advert_int 1 - authentication { - auth_type PASS - auth_pass k8s-test - } - virtual_ipaddress { - 192.168.122.59 - } - track_script { - check_harbor - } -} diff --git a/scripts/harbor_keepalived/keepalived-master.conf b/scripts/harbor_keepalived/keepalived-master.conf deleted file mode 100644 index de3566e4..00000000 --- a/scripts/harbor_keepalived/keepalived-master.conf +++ /dev/null @@ -1,31 +0,0 @@ -vrrp_script check_harbor { - script "/srv/conf/keepalived/check_harbor.sh" - interval 10 # 间隔时间，单位为秒，默认1秒 - fall 2 # 脚本几次失败转换为失败 - rise 2 # 脚本连续监测成功后，把服务器从失败标记为成功的次数 - timeout 5 - init_fail -} -global_defs { - script_user root - router_id harbor-ha - enable_script_security - lvs_sync_daemon ens3 VI_1 -} -vrrp_instance VI_1 { - state MASTER - interface ens3 - virtual_router_id 31 # 如果同一个局域网中有多套keepalive，那么要保证该id唯一 - priority 100 - advert_int 1 - authentication { - auth_type PASS - auth_pass k8s-test - } - virtual_ipaddress { - 192.168.122.59 - } - track_script { - check_harbor - } -} diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml deleted file mode 100644 index 88ba40d2..00000000 --- a/templates/cluster-template.yaml +++ /dev/null @@ -1,182 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: '${CLUSTER_NAME}' -spec: - clusterNetwork: - services: - cidrBlocks: ["10.233.0.0/18"] - pods: - cidrBlocks: ["10.233.64.0/18"] - serviceDomain: "cluster.local" - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: KKCluster - name: '${CLUSTER_NAME}' - controlPlaneRef: - kind: KubeadmControlPlane - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - name: "${CLUSTER_NAME}-control-plane" ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKCluster -metadata: - name: '${CLUSTER_NAME}' -spec: - component: - zone: '${KKZONE}' - nodes: - auth: - user: '${USER_NAME}' - password: '${PASSWORD}' - instances: ${INSTANCES} - controlPlaneLoadBalancer: - host: '${CONTROL_PLANE_ENDPOINT_IP}' ---- -kind: KubeadmControlPlane -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -metadata: - name: "${CLUSTER_NAME}-control-plane" -spec: - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - machineTemplate: - infrastructureRef: - kind: KKMachineTemplate - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - name: "${CLUSTER_NAME}-control-plane" - kubeadmConfigSpec: - files: - - content: | - apiVersion: v1 - kind: Pod - metadata: - name: kube-vip - namespace: kube-system - spec: - containers: - - args: - - manager - env: - - name: address - value: ${CONTROL_PLANE_ENDPOINT_IP} - - name: vip_interface - value: ${VIP_NETWORK_INTERFACE=""} - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: lb_enable - value: "true" - - name: lb_port - value: "6443" - image: ghcr.io/kube-vip/kube-vip:v0.5.0 - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - hostNetwork: true - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - volumes: - - hostPath: - path: /etc/kubernetes/admin.conf - type: FileOrCreate - name: kubeconfig - status: {} - owner: root:root - path: /etc/kubernetes/manifests/kube-vip.yaml - initConfiguration: - nodeRegistration: - criSocket: unix:///var/run/containerd/containerd.sock - joinConfiguration: - nodeRegistration: - criSocket: unix:///var/run/containerd/containerd.sock - version: "${KUBERNETES_VERSION}" ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKMachineTemplate -metadata: - name: "${CLUSTER_NAME}-control-plane" -spec: - template: - spec: - roles: - - control-plane - repository: - iso: "none" - update: false ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: "${CLUSTER_NAME}-md-0" -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: - template: - spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" - bootstrap: - configRef: - name: "${CLUSTER_NAME}-md-0" - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - infrastructureRef: - name: "${CLUSTER_NAME}-md-0" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: KKMachineTemplate ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKMachineTemplate -metadata: - name: '${CLUSTER_NAME}-md-0' -spec: - template: - spec: - roles: - - worker - repository: - iso: "none" - update: false ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: "${CLUSTER_NAME}-md-0" -spec: - template: - spec: - joinConfiguration: - nodeRegistration: - criSocket: unix:///var/run/containerd/containerd.sock diff --git a/test/e2e/.gitignore b/test/e2e/.gitignore deleted file mode 100644 index 6722cd96..00000000 --- a/test/e2e/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.xml diff --git a/test/e2e/Makefile b/test/e2e/Makefile deleted file mode 100644 index cb5b8289..00000000 --- a/test/e2e/Makefile +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2022 The KubeSphere Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -.DEFAULT_GOAL:=help - -GOPROXY := $(shell go env GOPROXY) -ifeq ($(GOPROXY),) -GOPROXY := https://goproxy.cn,direct -endif -export GOPROXY - -REPO_ROOT := $(shell git rev-parse --show-toplevel) - -help: ## Display this help - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-25s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -## -------------------------------------- -## Binaries -## -------------------------------------- - -TOOLS_DIR := $(REPO_ROOT)/hack/tools -BIN_DIR := bin -TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR) -GINGKO_VER := v1.16.5 -GINKGO_BIN := ginkgo -GINKGO := $(abspath $(TOOLS_BIN_DIR)/$(GINKGO_BIN)-$(GINGKO_VER)) -GINKGO_PKG := github.com/onsi/ginkgo/ginkgo -GO_INSTALL := $(REPO_ROOT)/scripts/go_install.sh -KUSTOMIZE_BIN := kustomize -KUSTOMIZE_VER := $(shell grep "^KUSTOMIZE_VER" $(REPO_ROOT)/Makefile | awk '{print $$NF}') -KUSTOMIZE := $(abspath $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER)) -KUSTOMIZE_PKG := sigs.k8s.io/kustomize/kustomize/v4 -KIND := $(abspath $(TOOLS_BIN_DIR)/$(KIND_BIN)) -KIND_BIN := kind -KIND_PKG := sigs.k8s.io/kind -KIND_VER := v0.14.0 - -$(GINKGO): # Build ginkgo from tools folder. - GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(GINKGO_PKG) $(GINKGO_BIN) $(GINGKO_VER) - -$(KUSTOMIZE): # Build kustomize from tools folder. - GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(KUSTOMIZE_PKG) $(KUSTOMIZE_BIN) $(KUSTOMIZE_VER) - -$(KIND): - GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(KIND_PKG) $(KIND_BIN) $(KIND_VER) - -.PHONY: $(GINKGO_BIN) -$(GINKGO_BIN): $(GINKGO) ## Build a local copy of ginkgo - -.PHONY: $(KUSTOMIZE_BIN) -$(KUSTOMIZE_BIN): $(KUSTOMIZE) ## Build a local copy of kustomize - -.PHONY: $(KIND_BIN) -$(KIND_BIN): $(KIND) ## Build a local copy of KIND - -## -------------------------------------- -## Templates -## -------------------------------------- - -KUBEKEY_TEMPLATES := $(REPO_ROOT)/test/e2e/data/infrastructure-kubekey -KUBEKEY_K3S_TEMPLATES := $(REPO_ROOT)/test/e2e/data/k3s - -.PHONY: cluster-templates -cluster-templates: $(KUSTOMIZE) cluster-templates-v1beta1 ## Generate cluster templates for all versions - -.PHONY: cluster-templates-k3s -cluster-templates-k3s: $(KUSTOMIZE) cluster-templates-k3s-v1beta1 - -cluster-templates-v1beta1: $(KUSTOMIZE) ## Generate cluster templates for v1beta1 - $(KUSTOMIZE) build $(KUBEKEY_TEMPLATES)/v1beta1/cluster-template --load-restrictor LoadRestrictionsNone > $(KUBEKEY_TEMPLATES)/v1beta1/cluster-template.yaml - -cluster-templates-k3s-v1beta1: $(KUSTOMIZE) - $(KUSTOMIZE) build $(KUBEKEY_K3S_TEMPLATES)/v1beta1/cluster-template --load-restrictor LoadRestrictionsNone > $(KUBEKEY_K3S_TEMPLATES)/v1beta1/cluster-template.yaml - -## -------------------------------------- -## Testing -## -------------------------------------- - -GINKGO_FOCUS ?= -GINKGO_SKIP ?= -GINKGO_NODES ?= 1 -E2E_CONF_FILE ?= ${REPO_ROOT}/test/e2e/config/e2e_conf.yaml -E2E_K3S_CONF_FILE ?= ${REPO_ROOT}/test/e2e/config/e2e_k3s_conf.yaml -ARTIFACTS ?= ${REPO_ROOT}/_artifacts -ARTIFACTS_K3S ?= ${REPO_ROOT}/_artifacts_k3s -SKIP_RESOURCE_CLEANUP ?= false -USE_EXISTING_CLUSTER ?= false -GINKGO_NOCOLOR ?= false - -# to set multiple ginkgo skip flags, if any -ifneq ($(strip $(GINKGO_SKIP)),) -_SKIP_ARGS := $(foreach arg,$(strip $(GINKGO_SKIP)),-skip="$(arg)") -endif - -.PHONY: run -run: $(GINKGO) $(KIND) cluster-templates ## Run the end-to-end tests - $(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" $(_SKIP_ARGS) -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) $(GINKGO_ARGS) ./suites/capkk/... -- \ - -e2e.artifacts-folder="$(ARTIFACTS)" \ - -e2e.config="$(E2E_CONF_FILE)" \ - -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) - -.PHONY: run-k3s -run-k3s: $(GINKGO) $(KIND) cluster-templates-k3s ## Run the end-to-end tests - $(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" $(_SKIP_ARGS) -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) $(GINKGO_ARGS) ./suites/k3s/... -- \ - -e2e.artifacts-folder="$(ARTIFACTS_K3S)" \ - -e2e.config="$(E2E_K3S_CONF_FILE)" \ - -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) \ No newline at end of file diff --git a/test/e2e/config/e2e_conf.yaml b/test/e2e/config/e2e_conf.yaml deleted file mode 100644 index 1b824ea9..00000000 --- a/test/e2e/config/e2e_conf.yaml +++ /dev/null @@ -1,120 +0,0 @@ ---- - -images: - # Use local dev images built source tree; - - name: kubespheredev/capkk-controller-{ARCH}:e2e - loadBehavior: mustLoad -# -## PLEASE KEEP THESE UP TO DATE WITH THE COMPONENTS - - # Cluster API v1beta1 Preloads - - name: quay.io/jetstack/cert-manager-cainjector:v1.8.2 - loadBehavior: tryLoad - - name: quay.io/jetstack/cert-manager-webhook:v1.8.2 - loadBehavior: tryLoad - - name: quay.io/jetstack/cert-manager-controller:v1.8.2 - loadBehavior: tryLoad - - name: k8s.gcr.io/cluster-api/cluster-api-controller:v1.2.1 - loadBehavior: tryLoad - - name: k8s.gcr.io/cluster-api/kubeadm-bootstrap-controller:v1.2.1 - loadBehavior: tryLoad - - name: k8s.gcr.io/cluster-api/kubeadm-control-plane-controller:v1.2.1 - loadBehavior: tryLoad - - name: ghcr.io/kube-vip/kube-vip:v0.5.0 - loadBehavior: tryLoad - -providers: - - name: cluster-api - type: CoreProvider - versions: - - name: v1.2.1 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.1/core-components.yaml" - type: "url" - contract: v1beta1 - files: - - sourcePath: "../data/shared/v1beta1/metadata.yaml" - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - old: --metrics-bind-addr=127.0.0.1:8080 - new: --metrics-bind-addr=:8080 - - name: kubeadm - type: BootstrapProvider - files: - - sourcePath: "../data/shared/v1beta1/metadata.yaml" - versions: - - name: v1.2.1 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.1/bootstrap-components.yaml" - type: "url" - contract: v1beta1 - files: - - sourcePath: "../data/shared/v1beta1/metadata.yaml" - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - old: --metrics-bind-addr=127.0.0.1:8080 - new: --metrics-bind-addr=:8080 - - name: kubeadm - type: ControlPlaneProvider - files: - - sourcePath: "../data/shared/v1beta1/metadata.yaml" - versions: - - name: v1.2.1 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.1/control-plane-components.yaml" - type: "url" - contract: v1beta1 - files: - - sourcePath: "../data/shared/v1beta1/metadata.yaml" - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - old: --metrics-bind-addr=127.0.0.1:8080 - new: --metrics-bind-addr=:8080 - - name: kubekey - type: InfrastructureProvider - versions: - - name: v3.0.0 - value: ../../../config/default - contract: v1beta1 - files: - - sourcePath: "../data/infrastructure-kubekey/v1beta1/cluster-template.yaml" - - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml" - replacements: - - old: docker.io/kubespheredev/capkk-controller:main - new: docker.io/kubespheredev/capkk-controller-amd64:e2e - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - -variables: - # Default variables for the e2e test; those values could be overridden via env variables, thus - # allowing the same e2e config file to be re-used in different Prow jobs e.g. each one with a K8s version permutation. - # The following Kubernetes versions should be the latest versions with already published kindest/node images. - # This avoids building node images in the default case which improves the test duration significantly. - KUBERNETES_VERSION_MANAGEMENT: "v1.24.0" - KUBERNETES_VERSION: "v1.24.0" - IMAGE_REPOSITORY: "k8s.gcr.io" - CNI: "../../data/cni/calico.yaml" - EVENT_BRIDGE_INSTANCE_STATE: "true" - EXP_CLUSTER_RESOURCE_SET: "true" - IP_FAMILY: "IPv4" - SERVICE_CIDRS: "10.233.0.0/18" - POD_CIDRS: "10.233.64.0/18" - SERVICE_DOMAIN: "cluster.local" - KKZONE: "cn" - USER_NAME: "ubuntu" - PASSWORD: "Qcloud@123" - INSTANCES: "[{address: 192.168.100.3}, {address: 192.168.100.4}]" - CONTROL_PLANE_ENDPOINT_IP: "192.168.100.100" - CONTAINER_MANAGER_TYPE: containerd - CRI_SOCKET: unix:///var/run/containerd/containerd.sock - -intervals: - default/wait-controllers: [ "5m", "10s" ] - default/wait-cluster: [ "5m", "10s" ] - default/wait-control-plane: [ "30m", "10s" ] - default/wait-worker-nodes: [ "30m", "10s" ] - default/wait-delete-cluster: [ "5m", "10s" ] - default/wait-machine-upgrade: [ "15m", "1m" ] - default/wait-machine-remediation: [ "5m", "10s" ] - node-drain/wait-deployment-available: [ "3m", "10s" ] - node-drain/wait-machine-deleted: [ "2m", "10s" ] diff --git a/test/e2e/config/e2e_k3s_conf.yaml b/test/e2e/config/e2e_k3s_conf.yaml deleted file mode 100644 index 9b64246c..00000000 --- a/test/e2e/config/e2e_k3s_conf.yaml +++ /dev/null @@ -1,115 +0,0 @@ ---- - -images: - # Use local dev images built source tree; - - name: kubespheredev/capkk-controller-{ARCH}:e2e - loadBehavior: mustLoad - - name: kubespheredev/k3s-bootstrap-controller-{ARCH}:e2e - loadBehavior: mustLoad - - name: kubespheredev/k3s-control-plane-controller-{ARCH}:e2e - loadBehavior: mustLoad - # - ## PLEASE KEEP THESE UP TO DATE WITH THE COMPONENTS - - # Cluster API v1beta1 Preloads - - name: quay.io/jetstack/cert-manager-cainjector:v1.8.2 - loadBehavior: tryLoad - - name: quay.io/jetstack/cert-manager-webhook:v1.8.2 - loadBehavior: tryLoad - - name: quay.io/jetstack/cert-manager-controller:v1.8.2 - loadBehavior: tryLoad - - name: k8s.gcr.io/cluster-api/cluster-api-controller:v1.2.1 - loadBehavior: tryLoad - - name: ghcr.io/kube-vip/kube-vip:v0.5.0 - loadBehavior: tryLoad - -providers: - - name: cluster-api - type: CoreProvider - versions: - - name: v1.2.2 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.2/core-components.yaml" - type: "url" - contract: v1beta1 - files: - - sourcePath: "../data/shared/v1beta1/metadata.yaml" - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - old: --metrics-bind-addr=127.0.0.1:8080 - new: --metrics-bind-addr=:8080 - - name: k3s - type: BootstrapProvider - versions: - - name: v3.0.0 - value: ../../../bootstrap/k3s/config/default - contract: v1beta1 - files: - - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml" - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - old: --metrics-bind-addr=127.0.0.1:8080 - new: --metrics-bind-addr=:8080 - - old: docker.io/kubespheredev/k3s-bootstrap-controller:main - new: docker.io/kubespheredev/k3s-bootstrap-controller-amd64:e2e - - name: k3s - type: ControlPlaneProvider - versions: - - name: v3.0.0 - value: ../../../controlplane/k3s/config/default - contract: v1beta1 - files: - - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml" - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - old: --metrics-bind-addr=127.0.0.1:8080 - new: --metrics-bind-addr=:8080 - - old: docker.io/kubespheredev/k3s-control-plane-controller:main - new: docker.io/kubespheredev/k3s-control-plane-controller-amd64:e2e - - name: kubekey - type: InfrastructureProvider - versions: - - name: v3.0.0 - value: ../../../config/default - contract: v1beta1 - replacements: - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" - - old: docker.io/kubespheredev/capkk-controller:main - new: docker.io/kubespheredev/capkk-controller-amd64:e2e - files: - - sourcePath: "../data/k3s/v1beta1/cluster-template.yaml" - - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml" - -variables: - # Default variables for the e2e test; those values could be overridden via env variables, thus - # allowing the same e2e config file to be re-used in different Prow jobs e.g. each one with a K8s version permutation. - # The following Kubernetes versions should be the latest versions with already published kindest/node images. - # This avoids building node images in the default case which improves the test duration significantly. - KUBERNETES_VERSION_MANAGEMENT: "v1.24.0" - KUBERNETES_VERSION: "v1.24.7" - CNI: "../../data/cni/calico.yaml" - EVENT_BRIDGE_INSTANCE_STATE: "true" - EXP_CLUSTER_RESOURCE_SET: "true" - IP_FAMILY: "IPv4" - SERVICE_CIDRS: "10.233.0.0/18" - POD_CIDRS: "10.233.64.0/18" - SERVICE_DOMAIN: "cluster.local" - KKZONE: "" - USER_NAME: "ubuntu" - PASSWORD: "Qcloud@123" - INSTANCES: "[{address: 192.168.100.3}, {address: 192.168.100.4}, {address: 192.168.100.5}]" - CONTROL_PLANE_ENDPOINT_IP: "192.168.100.100" - -intervals: - default/wait-controllers: [ "5m", "10s" ] - default/wait-cluster: [ "5m", "10s" ] - default/wait-control-plane: [ "30m", "10s" ] - default/wait-worker-nodes: [ "30m", "10s" ] - default/wait-delete-cluster: [ "5m", "10s" ] - default/wait-machine-upgrade: [ "15m", "1m" ] - default/wait-machine-remediation: [ "5m", "10s" ] - node-drain/wait-deployment-available: [ "3m", "10s" ] - node-drain/wait-machine-deleted: [ "2m", "10s" ] diff --git a/test/e2e/data/cni/calico.yaml b/test/e2e/data/cni/calico.yaml deleted file mode 100644 index a10d0d41..00000000 --- a/test/e2e/data/cni/calico.yaml +++ /dev/null @@ -1,4664 +0,0 @@ ---- -# Source: calico/templates/calico-config.yaml (v3.23.2) -# This ConfigMap is used to configure a self-hosted Calico installation. -kind: ConfigMap -apiVersion: v1 -metadata: - name: calico-config - namespace: kube-system -data: - # Typha is disabled. - typha_service_name: "none" - # Configure the backend to use. - calico_backend: "bird" - - # Configure the MTU to use for workload interfaces and tunnels. - # By default, MTU is auto-detected, and explicitly setting this field should not be required. - # You can override auto-detection by providing a non-zero value. - veth_mtu: "0" - - # The CNI network configuration to install on each node. The special - # values in this config will be automatically populated. - cni_network_config: |- - { - "name": "k8s-pod-network", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "calico", - "log_level": "info", - "log_file_path": "/var/log/calico/cni/cni.log", - "datastore_type": "kubernetes", - "nodename": "__KUBERNETES_NODE_NAME__", - "mtu": __CNI_MTU__, - "ipam": { - "type": "calico-ipam" - }, - "policy": { - "type": "k8s" - }, - "kubernetes": { - "kubeconfig": "__KUBECONFIG_FILEPATH__" - } - }, - { - "type": "portmap", - "snat": true, - "capabilities": {"portMappings": true} - }, - { - "type": "bandwidth", - "capabilities": {"bandwidth": true} - } - ] - } - ---- -# Source: calico/templates/kdd-crds.yaml - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: bgpconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BGPConfiguration - listKind: BGPConfigurationList - plural: bgpconfigurations - singular: bgpconfiguration - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: BGPConfiguration contains the configuration for any BGP routing. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPConfigurationSpec contains the values of the BGP configuration. - properties: - asNumber: - description: 'ASNumber is the default AS number used by a node. [Default: - 64512]' - format: int32 - type: integer - bindMode: - description: BindMode indicates whether to listen for BGP connections - on all addresses (None) or only on the node's canonical IP address - Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen - for BGP connections on all addresses. - type: string - communities: - description: Communities is a list of BGP community values and their - arbitrary names for tagging routes. - items: - description: Community contains standard or large community value - and its name. - properties: - name: - description: Name given to community value. - type: string - value: - description: Value must be of format aa:nn or aa:nn:mm. - For standard community use aa:nn format, where aa and - nn are 16 bit number. For large community use aa:nn:mm - format, where aa, nn and mm are 32 bit number. Where, - aa is an AS Number, nn and mm are per-AS identifier. - pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ - type: string - type: object - type: array - listenPort: - description: ListenPort is the port where BGP protocol should listen. - Defaults to 179 - maximum: 65535 - minimum: 1 - type: integer - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: INFO]' - type: string - nodeMeshMaxRestartTime: - description: Time to allow for software restart for node-to-mesh peerings. When - specified, this is configured as the graceful restart timeout. When - not specified, the BIRD default of 120s is used. This field can - only be set on the default BGPConfiguration instance and requires - that NodeMesh is enabled - type: string - nodeMeshPassword: - description: Optional BGP password for full node-to-mesh peerings. - This field can only be set on the default BGPConfiguration instance - and requires that NodeMesh is enabled - properties: - secretKeyRef: - description: Selects a key of a secret in the node pod's namespace. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must be - defined - type: boolean - required: - - key - type: object - type: object - nodeToNodeMeshEnabled: - description: 'NodeToNodeMeshEnabled sets whether full node to node - BGP mesh is enabled. [Default: true]' - type: boolean - prefixAdvertisements: - description: PrefixAdvertisements contains per-prefix advertisement - configuration. - items: - description: PrefixAdvertisement configures advertisement properties - for the specified CIDR. - properties: - cidr: - description: CIDR for which properties should be advertised. - type: string - communities: - description: Communities can be list of either community names - already defined in Specs.Communities or community value - of format aa:nn or aa:nn:mm. For standard community use - aa:nn format, where aa and nn are 16 bit number. For - large community use aa:nn:mm format, where aa, nn and - mm are 32 bit number. Where,aa is an AS Number, nn and - mm are per-AS identifier. - items: - type: string - type: array - type: object - type: array - serviceClusterIPs: - description: ServiceClusterIPs are the CIDR blocks from which service - cluster IPs are allocated. If specified, Calico will advertise these - blocks, as well as any cluster IPs within them. - items: - description: ServiceClusterIPBlock represents a single allowed ClusterIP - CIDR block. - properties: - cidr: - type: string - type: object - type: array - serviceExternalIPs: - description: ServiceExternalIPs are the CIDR blocks for Kubernetes - Service External IPs. Kubernetes Service ExternalIPs will only be - advertised if they are within one of these blocks. - items: - description: ServiceExternalIPBlock represents a single allowed - External IP CIDR block. - properties: - cidr: - type: string - type: object - type: array - serviceLoadBalancerIPs: - description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes - Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress - IPs will only be advertised if they are within one of these blocks. - items: - description: ServiceLoadBalancerIPBlock represents a single allowed - LoadBalancer IP CIDR block. - properties: - cidr: - type: string - type: object - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: bgppeers.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BGPPeer - listKind: BGPPeerList - plural: bgppeers - singular: bgppeer - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPPeerSpec contains the specification for a BGPPeer resource. - properties: - asNumber: - description: The AS Number of the peer. - format: int32 - type: integer - keepOriginalNextHop: - description: Option to keep the original nexthop field when routes - are sent to a BGP Peer. Setting "true" configures the selected BGP - Peers node to use the "next hop keep;" instead of "next hop self;"(default) - in the specific branch of the Node on "bird.cfg". - type: boolean - maxRestartTime: - description: Time to allow for software restart. When specified, - this is configured as the graceful restart timeout. When not specified, - the BIRD default of 120s is used. - type: string - node: - description: The node name identifying the Calico node instance that - is targeted by this peer. If this is not set, and no nodeSelector - is specified, then this BGP peer selects all nodes in the cluster. - type: string - nodeSelector: - description: Selector for the nodes that should have this peering. When - this is set, the Node field must be empty. - type: string - numAllowedLocalASNumbers: - description: Maximum number of local AS numbers that are allowed in - the AS path for received routes. This removes BGP loop prevention - and should only be used if absolutely necesssary. - format: int32 - type: integer - password: - description: Optional BGP password for the peerings generated by this - BGPPeer resource. - properties: - secretKeyRef: - description: Selects a key of a secret in the node pod's namespace. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must be - defined - type: boolean - required: - - key - type: object - type: object - peerIP: - description: The IP address of the peer followed by an optional port - number to peer with. If port number is given, format should be []:port - or : for IPv4. If optional port number is not set, - and this peer IP and ASNumber belongs to a calico/node with ListenPort - set in BGPConfiguration, then we use that port to peer. - type: string - peerSelector: - description: Selector for the remote nodes to peer with. When this - is set, the PeerIP and ASNumber fields must be empty. For each - peering between the local node and selected remote nodes, we configure - an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, - and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The - remote AS number comes from the remote node's NodeBGPSpec.ASNumber, - or the global default if that is not set. - type: string - sourceAddress: - description: Specifies whether and how to configure a source address - for the peerings generated by this BGPPeer resource. Default value - "UseNodeIP" means to configure the node IP as the source address. "None" - means not to configure a source address. - type: string - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: blockaffinities.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BlockAffinity - listKind: BlockAffinityList - plural: blockaffinities - singular: blockaffinity - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BlockAffinitySpec contains the specification for a BlockAffinity - resource. - properties: - cidr: - type: string - deleted: - description: Deleted indicates that this block affinity is being deleted. - This field is a string for compatibility with older releases that - mistakenly treat this field as a string. - type: string - node: - type: string - state: - type: string - required: - - cidr - - deleted - - node - - state - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: caliconodestatuses.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: CalicoNodeStatus - listKind: CalicoNodeStatusList - plural: caliconodestatuses - singular: caliconodestatus - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: CalicoNodeStatusSpec contains the specification for a CalicoNodeStatus - resource. - properties: - classes: - description: Classes declares the types of information to monitor - for this calico/node, and allows for selective status reporting - about certain subsets of information. - items: - type: string - type: array - node: - description: The node name identifies the Calico node instance for - node status. - type: string - updatePeriodSeconds: - description: UpdatePeriodSeconds is the period at which CalicoNodeStatus - should be updated. Set to 0 to disable CalicoNodeStatus refresh. - Maximum update period is one day. - format: int32 - type: integer - type: object - status: - description: CalicoNodeStatusStatus defines the observed state of CalicoNodeStatus. - No validation needed for status since it is updated by Calico. - properties: - agent: - description: Agent holds agent status on the node. - properties: - birdV4: - description: BIRDV4 represents the latest observed status of bird4. - properties: - lastBootTime: - description: LastBootTime holds the value of lastBootTime - from bird.ctl output. - type: string - lastReconfigurationTime: - description: LastReconfigurationTime holds the value of lastReconfigTime - from bird.ctl output. - type: string - routerID: - description: Router ID used by bird. - type: string - state: - description: The state of the BGP Daemon. - type: string - version: - description: Version of the BGP daemon - type: string - type: object - birdV6: - description: BIRDV6 represents the latest observed status of bird6. - properties: - lastBootTime: - description: LastBootTime holds the value of lastBootTime - from bird.ctl output. - type: string - lastReconfigurationTime: - description: LastReconfigurationTime holds the value of lastReconfigTime - from bird.ctl output. - type: string - routerID: - description: Router ID used by bird. - type: string - state: - description: The state of the BGP Daemon. - type: string - version: - description: Version of the BGP daemon - type: string - type: object - type: object - bgp: - description: BGP holds node BGP status. - properties: - numberEstablishedV4: - description: The total number of IPv4 established bgp sessions. - type: integer - numberEstablishedV6: - description: The total number of IPv6 established bgp sessions. - type: integer - numberNotEstablishedV4: - description: The total number of IPv4 non-established bgp sessions. - type: integer - numberNotEstablishedV6: - description: The total number of IPv6 non-established bgp sessions. - type: integer - peersV4: - description: PeersV4 represents IPv4 BGP peers status on the node. - items: - description: CalicoNodePeer contains the status of BGP peers - on the node. - properties: - peerIP: - description: IP address of the peer whose condition we are - reporting. - type: string - since: - description: Since the state or reason last changed. - type: string - state: - description: State is the BGP session state. - type: string - type: - description: Type indicates whether this peer is configured - via the node-to-node mesh, or via en explicit global or - per-node BGPPeer object. - type: string - type: object - type: array - peersV6: - description: PeersV6 represents IPv6 BGP peers status on the node. - items: - description: CalicoNodePeer contains the status of BGP peers - on the node. - properties: - peerIP: - description: IP address of the peer whose condition we are - reporting. - type: string - since: - description: Since the state or reason last changed. - type: string - state: - description: State is the BGP session state. - type: string - type: - description: Type indicates whether this peer is configured - via the node-to-node mesh, or via en explicit global or - per-node BGPPeer object. - type: string - type: object - type: array - required: - - numberEstablishedV4 - - numberEstablishedV6 - - numberNotEstablishedV4 - - numberNotEstablishedV6 - type: object - lastUpdated: - description: LastUpdated is a timestamp representing the server time - when CalicoNodeStatus object last updated. It is represented in - RFC3339 form and is in UTC. - format: date-time - nullable: true - type: string - routes: - description: Routes reports routes known to the Calico BGP daemon - on the node. - properties: - routesV4: - description: RoutesV4 represents IPv4 routes on the node. - items: - description: CalicoNodeRoute contains the status of BGP routes - on the node. - properties: - destination: - description: Destination of the route. - type: string - gateway: - description: Gateway for the destination. - type: string - interface: - description: Interface for the destination - type: string - learnedFrom: - description: LearnedFrom contains information regarding - where this route originated. - properties: - peerIP: - description: If sourceType is NodeMesh or BGPPeer, IP - address of the router that sent us this route. - type: string - sourceType: - description: Type of the source where a route is learned - from. - type: string - type: object - type: - description: Type indicates if the route is being used for - forwarding or not. - type: string - type: object - type: array - routesV6: - description: RoutesV6 represents IPv6 routes on the node. - items: - description: CalicoNodeRoute contains the status of BGP routes - on the node. - properties: - destination: - description: Destination of the route. - type: string - gateway: - description: Gateway for the destination. - type: string - interface: - description: Interface for the destination - type: string - learnedFrom: - description: LearnedFrom contains information regarding - where this route originated. - properties: - peerIP: - description: If sourceType is NodeMesh or BGPPeer, IP - address of the router that sent us this route. - type: string - sourceType: - description: Type of the source where a route is learned - from. - type: string - type: object - type: - description: Type indicates if the route is being used for - forwarding or not. - type: string - type: object - type: array - type: object - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: clusterinformations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: ClusterInformation - listKind: ClusterInformationList - plural: clusterinformations - singular: clusterinformation - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: ClusterInformation contains the cluster specific information. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ClusterInformationSpec contains the values of describing - the cluster. - properties: - calicoVersion: - description: CalicoVersion is the version of Calico that the cluster - is running - type: string - clusterGUID: - description: ClusterGUID is the GUID of the cluster - type: string - clusterType: - description: ClusterType describes the type of the cluster - type: string - datastoreReady: - description: DatastoreReady is used during significant datastore migrations - to signal to components such as Felix that it should wait before - accessing the datastore. - type: boolean - variant: - description: Variant declares which variant of Calico should be active. - type: string - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: felixconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: FelixConfiguration - listKind: FelixConfigurationList - plural: felixconfigurations - singular: felixconfiguration - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: Felix Configuration contains the configuration for Felix. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: FelixConfigurationSpec contains the values of the Felix configuration. - properties: - allowIPIPPacketsFromWorkloads: - description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads - [Default: false]' - type: boolean - allowVXLANPacketsFromWorkloads: - description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads - [Default: false]' - type: boolean - awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enable" or "Disable". [Default: - DoNothing]' - enum: - - DoNothing - - Enable - - Disable - type: string - bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. [Default: - true]' - type: boolean - bpfDataIfacePattern: - description: BPFDataIfacePattern is a regular expression that controls - which interfaces Felix should attach BPF programs to in order to - catch traffic to/from the network. This needs to match the interfaces - that Calico workload traffic flows over as well as any interfaces - that handle incoming traffic to nodeports and services from outside - the cluster. It should not match the workload interfaces (usually - named cali...). - type: string - bpfDisableUnprivileged: - description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled - sysctl to disable unprivileged use of BPF. This ensures that unprivileged - users cannot access Calico''s BPF maps and cannot insert their own - BPF programs to interfere with Calico''s. [Default: true]' - type: boolean - bpfEnabled: - description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. - [Default: false]' - type: boolean - bpfEnforceRPF: - description: 'BPFEnforceRPF enforce strict RPF on all interfaces with - BPF programs regardless of what is the per-interfaces or global - setting. Possible values are Disabled or Strict. [Default: Strict]' - type: string - bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit - mark that is set on connections from an external client to a local - service. This mark allows us to control how packets of that connection - are routed within the host and how is routing intepreted by RPF - check. [Default: 0]' - type: integer - bpfExternalServiceMode: - description: 'BPFExternalServiceMode in BPF mode, controls how connections - from outside the cluster to services (node ports and cluster IPs) - are forwarded to remote workloads. If set to "Tunnel" then both - request and response traffic is tunneled to the remote node. If - set to "DSR", the request traffic is tunneled but the response traffic - is sent directly from the remote node. In "DSR" mode, the remote - node appears to use the IP of the ingress node; this requires a - permissive L2 network. [Default: Tunnel]' - type: string - bpfKubeProxyEndpointSlicesEnabled: - description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls - whether Felix's embedded kube-proxy accepts EndpointSlices or not. - type: boolean - bpfKubeProxyIptablesCleanupEnabled: - description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF - mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s - iptables chains. Should only be enabled if kube-proxy is not running. [Default: - true]' - type: boolean - bpfKubeProxyMinSyncPeriod: - description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the - minimum time between updates to the dataplane for Felix''s embedded - kube-proxy. Lower values give reduced set-up latency. Higher values - reduce Felix CPU usage by batching up more work. [Default: 1s]' - type: string - bpfLogLevel: - description: 'BPFLogLevel controls the log level of the BPF programs - when in BPF dataplane mode. One of "Off", "Info", or "Debug". The - logs are emitted to the BPF trace pipe, accessible with the command - tc exec bpf debug. [Default: Off].' - type: string - bpfMapSizeConntrack: - description: 'BPFMapSizeConntrack sets the size for the conntrack - map. This map must be large enough to hold an entry for each active - connection. Warning: changing the size of the conntrack map can - cause disruption.' - type: integer - bpfMapSizeIPSets: - description: BPFMapSizeIPSets sets the size for ipsets map. The IP - sets map must be large enough to hold an entry for each endpoint - matched by every selector in the source/destination matches in network - policy. Selectors such as "all()" can result in large numbers of - entries (one entry per endpoint in that case). - type: integer - bpfMapSizeNATAffinity: - type: integer - bpfMapSizeNATBackend: - description: BPFMapSizeNATBackend sets the size for nat back end map. - This is the total number of endpoints. This is mostly more than - the size of the number of services. - type: integer - bpfMapSizeNATFrontend: - description: BPFMapSizeNATFrontend sets the size for nat front end - map. FrontendMap should be large enough to hold an entry for each - nodeport, external IP and each port in each service. - type: integer - bpfMapSizeRoute: - description: BPFMapSizeRoute sets the size for the routes map. The - routes map should be large enough to hold one entry per workload - and a handful of entries per host (enough to cover its own IPs and - tunnel IPs). - type: integer - bpfPSNATPorts: - anyOf: - - type: integer - - type: string - description: 'BPFPSNATPorts sets the range from which we randomly - pick a port if there is a source port collision. This should be - within the ephemeral range as defined by RFC 6056 (1024–65535) and - preferably outside the ephemeral ranges used by common operating - systems. Linux uses 32768–60999, while others mostly use the IANA - defined range 49152–65535. It is not necessarily a problem if this - range overlaps with the operating systems. Both ends of the range - are inclusive. [Default: 20000:29999]' - pattern: ^.* - x-kubernetes-int-or-string: true - chainInsertMode: - description: 'ChainInsertMode controls whether Felix hooks the kernel''s - top-level iptables chains by inserting a rule at the top of the - chain or by appending a rule at the bottom. insert is the safe default - since it prevents Calico''s rules from being bypassed. If you switch - to append mode, be sure that the other rules in the chains signal - acceptance by falling through to the Calico rules, otherwise the - Calico policy will be bypassed. [Default: insert]' - type: string - dataplaneDriver: - description: DataplaneDriver filename of the external dataplane driver - to use. Only used if UseInternalDataplaneDriver is set to false. - type: string - dataplaneWatchdogTimeout: - description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout - used for Felix''s (internal) dataplane driver. Increase this value - if you experience spurious non-ready or non-live events when Felix - is under heavy load. Decrease the value to get felix to report non-live - or non-ready more quickly. [Default: 90s]' - type: string - debugDisableLogDropping: - type: boolean - debugMemoryProfilePath: - type: string - debugSimulateCalcGraphHangAfter: - type: string - debugSimulateDataplaneHangAfter: - type: string - defaultEndpointToHostAction: - description: 'DefaultEndpointToHostAction controls what happens to - traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables "DROP" action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables "INPUT" chain; - Calico will insert its rules at the top of that chain, then "RETURN" - packets to the "INPUT" chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' - type: string - deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. - type: integer - deviceRouteSourceAddress: - description: This is the IPv4 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. - type: string - deviceRouteSourceAddressIPv6: - description: This is the IPv6 source address to use on programmed - device routes. By default the source address is left blank, leaving - the kernel to choose the source address used. - type: string - disableConntrackInvalidCheck: - type: boolean - endpointReportingDelay: - type: string - endpointReportingEnabled: - type: boolean - externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. - items: - type: string - type: array - failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports - and CIDRs that Felix will allow incoming traffic to host endpoints - on irrespective of the security policy. This is useful to avoid - accidentally cutting off a host with incorrect configuration. For - back-compatibility, if the protocol is not specified, it defaults - to "tcp". If a CIDR is not specified, it will allow traffic from - all addresses. To disable all inbound host ports, use the value - none. The default value allows ssh access and DHCP. [Default: tcp:22, - udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' - items: - description: ProtoPort is combination of protocol, port, and CIDR. - Protocol and port must be specified. - properties: - net: - type: string - port: - type: integer - protocol: - type: string - required: - - port - - protocol - type: object - type: array - failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports - and CIDRs that Felix will allow outgoing traffic from host endpoints - to irrespective of the security policy. This is useful to avoid - accidentally cutting off a host with incorrect configuration. For - back-compatibility, if the protocol is not specified, it defaults - to "tcp". If a CIDR is not specified, it will allow traffic from - all addresses. To disable all outbound host ports, use the value - none. The default value opens etcd''s standard ports to ensure that - Felix does not get cut off from etcd as well as allowing DHCP and - DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, - tcp:6667, udp:53, udp:67]' - items: - description: ProtoPort is combination of protocol, port, and CIDR. - Protocol and port must be specified. - properties: - net: - type: string - port: - type: integer - protocol: - type: string - required: - - port - - protocol - type: object - type: array - featureDetectOverride: - description: FeatureDetectOverride is used to override the feature - detection. Values are specified in a comma separated list with no - spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". - "true" or "false" will force the feature, empty or omitted values - are auto-detected. - type: string - floatingIPs: - default: Disabled - description: FloatingIPs configures whether or not Felix will program - floating IP addresses. - enum: - - Enabled - - Disabled - type: string - genericXDPEnabled: - description: 'GenericXDPEnabled enables Generic XDP so network cards - that don''t support XDP offload or driver modes can use XDP. This - is not recommended since it doesn''t provide better performance - than iptables. [Default: false]' - type: boolean - healthEnabled: - type: boolean - healthHost: - type: string - healthPort: - type: integer - interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' - type: string - interfacePrefix: - description: 'InterfacePrefix is the interface name prefix that identifies - workload endpoints and so distinguishes them from host endpoint - interfaces. Note: in environments other than bare metal, the orchestrators - configure this appropriately. For example our Kubernetes and Docker - integrations set the ''cali'' value, and our OpenStack integration - sets the ''tap'' value. [Default: cali]' - type: string - interfaceRefreshInterval: - description: InterfaceRefreshInterval is the period at which Felix - rescans local interfaces to verify their state. The rescan can be - disabled by setting the interval to 0. - type: string - ipipEnabled: - description: 'IPIPEnabled overrides whether Felix should configure - an IPIP interface on the host. Optional as Felix determines this - based on the existing IP pools. [Default: nil (unset)]' - type: boolean - ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' - type: integer - ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' - type: string - iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is legacy. - type: string - iptablesFilterAllowAction: - type: string - iptablesLockFilePath: - description: 'IptablesLockFilePath is the location of the iptables - lock file. You may need to change this if the lock file is not in - its standard location (for example if you have mapped it into Felix''s - container at a different path). [Default: /run/xtables.lock]' - type: string - iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' - type: string - iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' - type: string - iptablesMangleAllowAction: - type: string - iptablesMarkMask: - description: 'IptablesMarkMask is the mask that Felix selects its - IPTables Mark bits from. Should be a 32 bit hexadecimal number with - at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' - format: int32 - type: integer - iptablesNATOutgoingInterfaceFilter: - type: string - iptablesPostWriteCheckInterval: - description: 'IptablesPostWriteCheckInterval is the period after Felix - has done a write to the dataplane that it schedules an extra read - back in order to check the write was not clobbered by another process. - This should only occur if another application on the system doesn''t - respect the iptables lock. [Default: 1s]' - type: string - iptablesRefreshInterval: - description: 'IptablesRefreshInterval is the period at which Felix - re-checks the IP sets in the dataplane to ensure that no other process - has accidentally broken Calico''s rules. Set to 0 to disable IP - sets refresh. Note: the default for this value is lower than the - other refresh intervals as a workaround for a Linux kernel bug that - was fixed in kernel version 4.11. If you are using v4.11 or greater - you may want to set this to, a higher value to reduce Felix CPU - usage. [Default: 10s]' - type: string - ipv6Support: - description: IPv6Support controls whether Felix enables support for - IPv6 (if supported by the in-use dataplane). - type: boolean - kubeNodePortRanges: - description: 'KubeNodePortRanges holds list of port ranges used for - service node ports. Only used if felix detects kube-proxy running - in ipvs mode. Felix uses these ranges to separate host and workload - traffic. [Default: 30000:32767].' - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - logDebugFilenameRegex: - description: LogDebugFilenameRegex controls which source code files - have their Debug log output included in the logs. Only logs from - files with names that match the given regular expression are included. The - filter only applies to Debug level logs. - type: string - logFilePath: - description: 'LogFilePath is the full path to the Felix log. Set to - none to disable file logging. [Default: /var/log/calico/felix.log]' - type: string - logPrefix: - description: 'LogPrefix is the log prefix that Felix uses when rendering - LOG rules. [Default: calico-packet]' - type: string - logSeverityFile: - description: 'LogSeverityFile is the log severity above which logs - are sent to the log file. [Default: Info]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: Info]' - type: string - logSeveritySys: - description: 'LogSeveritySys is the log severity above which logs - are sent to the syslog. Set to None for no logging to syslog. [Default: - Info]' - type: string - maxIpsetSize: - type: integer - metadataAddr: - description: 'MetadataAddr is the IP address or domain name of the - server that can answer VM queries for cloud-init metadata. In OpenStack, - this corresponds to the machine running nova-api (or in Ubuntu, - nova-api-metadata). A value of none (case insensitive) means that - Felix should not set up any NAT rule for the metadata path. [Default: - 127.0.0.1]' - type: string - metadataPort: - description: 'MetadataPort is the port of the metadata server. This, - combined with global.MetadataAddr (if not ''None''), is used to - set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. - In most cases this should not need to be changed [Default: 8775].' - type: integer - mtuIfacePattern: - description: MTUIfacePattern is a regular expression that controls - which interfaces Felix should scan in order to calculate the host's - MTU. This should not match workload interfaces (usually named cali...). - type: string - natOutgoingAddress: - description: NATOutgoingAddress specifies an address to use when performing - source NAT for traffic in a natOutgoing pool that is leaving the - network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) - type: string - natPortRange: - anyOf: - - type: integer - - type: string - description: NATPortRange specifies the range of ports that is used - for port mapping when doing outgoing NAT. When unset the default - behavior of the network stack is used. - pattern: ^.* - x-kubernetes-int-or-string: true - netlinkTimeout: - type: string - openstackRegion: - description: 'OpenstackRegion is the name of the region that a particular - Felix belongs to. In a multi-region Calico/OpenStack deployment, - this must be configured somehow for each Felix (here in the datamodel, - or in felix.cfg or the environment on each compute node), and must - match the [calico] openstack_region value configured in neutron.conf - on each node. [Default: Empty]' - type: string - policySyncPathPrefix: - description: 'PolicySyncPathPrefix is used to by Felix to communicate - policy changes to external services, like Application layer policy. - [Default: Empty]' - type: string - prometheusGoMetricsEnabled: - description: 'PrometheusGoMetricsEnabled disables Go runtime metrics - collection, which the Prometheus client does by default, when set - to false. This reduces the number of metrics reported, reducing - Prometheus load. [Default: true]' - type: boolean - prometheusMetricsEnabled: - description: 'PrometheusMetricsEnabled enables the Prometheus metrics - server in Felix if set to true. [Default: false]' - type: boolean - prometheusMetricsHost: - description: 'PrometheusMetricsHost is the host that the Prometheus - metrics server should bind to. [Default: empty]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. [Default: 9091]' - type: integer - prometheusProcessMetricsEnabled: - description: 'PrometheusProcessMetricsEnabled disables process metrics - collection, which the Prometheus client does by default, when set - to false. This reduces the number of metrics reported, reducing - Prometheus load. [Default: true]' - type: boolean - prometheusWireGuardMetricsEnabled: - description: 'PrometheusWireGuardMetricsEnabled disables wireguard - metrics collection, which the Prometheus client does by default, - when set to false. This reduces the number of metrics reported, - reducing Prometheus load. [Default: true]' - type: boolean - removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. - type: boolean - reportingInterval: - description: 'ReportingInterval is the interval at which Felix reports - its status into the datastore or 0 to disable. Must be non-zero - in OpenStack deployments. [Default: 30s]' - type: string - reportingTTL: - description: 'ReportingTTL is the time-to-live setting for process-wide - status reports. [Default: 90s]' - type: string - routeRefreshInterval: - description: 'RouteRefreshInterval is the period at which Felix re-checks - the routes in the dataplane to ensure that no other process has - accidentally broken Calico''s rules. Set to 0 to disable route refresh. - [Default: 90s]' - type: string - routeSource: - description: 'RouteSource configures where Felix gets its routing - information. - WorkloadIPs: use workload endpoints to construct - routes. - CalicoIPAM: the default - use IPAM data to construct routes.' - type: string - routeTableRange: - description: Deprecated in favor of RouteTableRanges. Calico programs - additional Linux route tables for various purposes. RouteTableRange - specifies the indices of the route tables that Calico should use. - properties: - max: - type: integer - min: - type: integer - required: - - max - - min - type: object - routeTableRanges: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRanges specifies a set of table index ranges - that Calico should use. DeprecatesRouteTableRange, overrides RouteTableRange. - items: - properties: - max: - type: integer - min: - type: integer - required: - - max - - min - type: object - type: array - serviceLoopPrevention: - description: 'When service IP advertisement is enabled, prevent routing - loops to service IPs that are not in use, by dropping or rejecting - packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", - in which case such routing loops continue to be allowed. [Default: - Drop]' - type: string - sidecarAccelerationEnabled: - description: 'SidecarAccelerationEnabled enables experimental sidecar - acceleration [Default: false]' - type: boolean - usageReportingEnabled: - description: 'UsageReportingEnabled reports anonymous Calico version - number and cluster size to projectcalico.org. Logs warnings returned - by the usage server. For example, if a significant security vulnerability - has been discovered in the version of Calico being used. [Default: - true]' - type: boolean - usageReportingInitialDelay: - description: 'UsageReportingInitialDelay controls the minimum delay - before Felix makes a report. [Default: 300s]' - type: string - usageReportingInterval: - description: 'UsageReportingInterval controls the interval at which - Felix makes reports. [Default: 86400s]' - type: string - useInternalDataplaneDriver: - description: UseInternalDataplaneDriver, if true, Felix will use its - internal dataplane programming logic. If false, it will launch - an external dataplane driver and communicate with it over protobuf. - type: boolean - vxlanEnabled: - description: 'VXLANEnabled overrides whether Felix should create the - VXLAN tunnel device for VXLAN networking. Optional as Felix determines - this based on the existing IP pools. [Default: nil (unset)]' - type: boolean - vxlanMTU: - description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel - device. See Configuring MTU [Default: 1410]' - type: integer - vxlanMTUV6: - description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel - device. See Configuring MTU [Default: 1390]' - type: integer - vxlanPort: - type: integer - vxlanVNI: - type: integer - wireguardEnabled: - description: 'WireguardEnabled controls whether Wireguard is enabled. - [Default: false]' - type: boolean - wireguardHostEncryptionEnabled: - description: 'WireguardHostEncryptionEnabled controls whether Wireguard - host-to-host encryption is enabled. [Default: false]' - type: boolean - wireguardInterfaceName: - description: 'WireguardInterfaceName specifies the name to use for - the Wireguard interface. [Default: wg.calico]' - type: string - wireguardKeepAlive: - description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive - option. Set 0 to disable. [Default: 0]' - type: string - wireguardListeningPort: - description: 'WireguardListeningPort controls the listening port used - by Wireguard. [Default: 51820]' - type: integer - wireguardMTU: - description: 'WireguardMTU controls the MTU on the Wireguard interface. - See Configuring MTU [Default: 1420]' - type: integer - wireguardRoutingRulePriority: - description: 'WireguardRoutingRulePriority controls the priority value - to use for the Wireguard routing rule. [Default: 99]' - type: integer - workloadSourceSpoofing: - description: WorkloadSourceSpoofing controls whether pods can use - the allowedSourcePrefixes annotation to send traffic with a source - IP address that is not theirs. This is disabled by default. When - set to "Any", pods can request any prefix. - type: string - xdpEnabled: - description: 'XDPEnabled enables XDP acceleration for suitable untracked - incoming deny rules. [Default: true]' - type: boolean - xdpRefreshInterval: - description: 'XDPRefreshInterval is the period at which Felix re-checks - all XDP state to ensure that no other process has accidentally broken - Calico''s BPF maps or attached programs. Set to 0 to disable XDP - refresh. [Default: 90s]' - type: string - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: globalnetworkpolicies.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: GlobalNetworkPolicy - listKind: GlobalNetworkPolicyList - plural: globalnetworkpolicies - singular: globalnetworkpolicy - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - applyOnForward: - description: ApplyOnForward indicates to apply the rules in this policy - on forward traffic. - type: boolean - doNotTrack: - description: DoNotTrack indicates whether packets matched by the rules - in this policy should go through the data plane's connection tracking, - such as Linux conntrack. If True, the rules in this policy are - applied before any data plane connection tracking, and packets allowed - by this policy are marked as not to be tracked. - type: boolean - egress: - description: The ordered set of egress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with \"Not\". All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a exact or a prefix match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - ingress: - description: The ordered set of ingress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with \"Not\". All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a exact or a prefix match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - namespaceSelector: - description: NamespaceSelector is an optional field for an expression - used to select a pod based on namespaces. - type: string - order: - description: Order is an optional field that specifies the order in - which the policy is applied. Policies with higher "order" are applied - after those with lower order. If the order is omitted, it may be - considered to be "infinite" - i.e. the policy will be applied last. Policies - with identical order will be applied in alphanumerical order based - on the Policy "Name". - type: number - preDNAT: - description: PreDNAT indicates to apply the rules in this policy before - any DNAT. - type: boolean - selector: - description: "The selector is an expression used to pick pick out - the endpoints that the policy should be applied to. \n Selector - expressions follow this syntax: \n \tlabel == \"string_literal\" - \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" - \ -> not equal; also matches if label is not present \tlabel in - { \"a\", \"b\", \"c\", ... } -> true if the value of label X is - one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", - ... } -> true if the value of label X is not one of \"a\", \"b\", - \"c\" \thas(label_name) -> True if that label is present \t! expr - -> negation of expr \texpr && expr -> Short-circuit and \texpr - || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() - or the empty selector -> matches all endpoints. \n Label names are - allowed to contain alphanumerics, -, _ and /. String literals are - more permissive but they do not support escape characters. \n Examples - (with made-up labels): \n \ttype == \"webserver\" && deployment - == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != - \"dev\" \t! has(label_name)" - type: string - serviceAccountSelector: - description: ServiceAccountSelector is an optional field for an expression - used to select a pod based on service accounts. - type: string - types: - description: "Types indicates whether this policy applies to ingress, - or to egress, or to both. When not explicitly specified (and so - the value on creation is empty or nil), Calico defaults Types according - to what Ingress and Egress rules are present in the policy. The - default is: \n - [ PolicyTypeIngress ], if there are no Egress rules - (including the case where there are also no Ingress rules) \n - - [ PolicyTypeEgress ], if there are Egress rules but no Ingress - rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are - both Ingress and Egress rules. \n When the policy is read back again, - Types will always be one of these values, never empty or nil." - items: - description: PolicyType enumerates the possible values of the PolicySpec - Types field. - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: globalnetworksets.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: GlobalNetworkSet - listKind: GlobalNetworkSetList - plural: globalnetworksets - singular: globalnetworkset - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs - that share labels to allow rules to refer to them via selectors. The labels - of GlobalNetworkSet are not namespaced. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: GlobalNetworkSetSpec contains the specification for a NetworkSet - resource. - properties: - nets: - description: The list of IP networks that belong to this set. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: hostendpoints.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: HostEndpoint - listKind: HostEndpointList - plural: hostendpoints - singular: hostendpoint - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: HostEndpointSpec contains the specification for a HostEndpoint - resource. - properties: - expectedIPs: - description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. - If \"InterfaceName\" is not present, Calico will look for an interface - matching any of the IPs in the list and apply policy to that. Note: - \tWhen using the selector match criteria in an ingress or egress - security Policy \tor Profile, Calico converts the selector into - a set of IP addresses. For host \tendpoints, the ExpectedIPs field - is used for that purpose. (If only the interface \tname is specified, - Calico does not learn the IPs of the interface for use in match - \tcriteria.)" - items: - type: string - type: array - interfaceName: - description: "Either \"*\", or the name of a specific Linux interface - to apply policy to; or empty. \"*\" indicates that this HostEndpoint - governs all traffic to, from or through the default network namespace - of the host named by the \"Node\" field; entering and leaving that - namespace via any interface, including those from/to non-host-networked - local workloads. \n If InterfaceName is not \"*\", this HostEndpoint - only governs traffic that enters or leaves the host through the - specific interface named by InterfaceName, or - when InterfaceName - is empty - through the specific interface that has one of the IPs - in ExpectedIPs. Therefore, when InterfaceName is empty, at least - one expected IP must be specified. Only external interfaces (such - as \"eth0\") are supported here; it isn't possible for a HostEndpoint - to protect traffic through a specific local workload interface. - \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; - initially just pre-DNAT policy. Please check Calico documentation - for the latest position." - type: string - node: - description: The node name identifying the Calico node instance. - type: string - ports: - description: Ports contains the endpoint's named ports, which may - be referenced in security policy rules. - items: - properties: - name: - type: string - port: - type: integer - protocol: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - required: - - name - - port - - protocol - type: object - type: array - profiles: - description: A list of identifiers of security Profile objects that - apply to this endpoint. Each profile is applied in the order that - they appear in this list. Profile rules are applied after the selector-based - security policy. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ipamblocks.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMBlock - listKind: IPAMBlockList - plural: ipamblocks - singular: ipamblock - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMBlockSpec contains the specification for an IPAMBlock - resource. - properties: - affinity: - description: Affinity of the block, if this block has one. If set, - it will be of the form "host:". If not set, this block - is not affine to a host. - type: string - allocations: - description: Array of allocations in-use within this block. nil entries - mean the allocation is free. For non-nil entries at index i, the - index is the ordinal of the allocation within this block and the - value is the index of the associated attributes in the Attributes - array. - items: - type: integer - # TODO: This nullable is manually added in. We should update controller-gen - # to handle []*int properly itself. - nullable: true - type: array - attributes: - description: Attributes is an array of arbitrary metadata associated - with allocations in the block. To find attributes for a given allocation, - use the value of the allocation's entry in the Allocations array - as the index of the element in this array. - items: - properties: - handle_id: - type: string - secondary: - additionalProperties: - type: string - type: object - type: object - type: array - cidr: - description: The block's CIDR. - type: string - deleted: - description: Deleted is an internal boolean used to workaround a limitation - in the Kubernetes API whereby deletion will not return a conflict - error if the block has been updated. It should not be set manually. - type: boolean - sequenceNumber: - default: 0 - description: We store a sequence number that is updated each time - the block is written. Each allocation will also store the sequence - number of the block at the time of its creation. When releasing - an IP, passing the sequence number associated with the allocation - allows us to protect against a race condition and ensure the IP - hasn't been released and re-allocated since the release request. - format: int64 - type: integer - sequenceNumberForAllocation: - additionalProperties: - format: int64 - type: integer - description: Map of allocated ordinal within the block to sequence - number of the block at the time of allocation. Kubernetes does not - allow numerical keys for maps, so the key is cast to a string. - type: object - strictAffinity: - description: StrictAffinity on the IPAMBlock is deprecated and no - longer used by the code. Use IPAMConfig StrictAffinity instead. - type: boolean - unallocated: - description: Unallocated is an ordered list of allocations which are - free in the block. - items: - type: integer - type: array - required: - - allocations - - attributes - - cidr - - strictAffinity - - unallocated - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ipamconfigs.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMConfig - listKind: IPAMConfigList - plural: ipamconfigs - singular: ipamconfig - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMConfigSpec contains the specification for an IPAMConfig - resource. - properties: - autoAllocateBlocks: - type: boolean - maxBlocksPerHost: - description: MaxBlocksPerHost, if non-zero, is the max number of blocks - that can be affine to each host. - type: integer - strictAffinity: - type: boolean - required: - - autoAllocateBlocks - - strictAffinity - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ipamhandles.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMHandle - listKind: IPAMHandleList - plural: ipamhandles - singular: ipamhandle - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMHandleSpec contains the specification for an IPAMHandle - resource. - properties: - block: - additionalProperties: - type: integer - type: object - deleted: - type: boolean - handleID: - type: string - required: - - block - - handleID - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ippools.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPPool - listKind: IPPoolList - plural: ippools - singular: ippool - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPPoolSpec contains the specification for an IPPool resource. - properties: - allowedUses: - description: AllowedUse controls what the IP pool will be used for. If - not specified or empty, defaults to ["Tunnel", "Workload"] for back-compatibility - items: - type: string - type: array - blockSize: - description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 122 for IPv6. - type: integer - cidr: - description: The pool CIDR. - type: string - disableBGPExport: - description: 'Disable exporting routes from this IP Pool''s CIDR over - BGP. [Default: false]' - type: boolean - disabled: - description: When disabled is true, Calico IPAM will not assign addresses - from this pool. - type: boolean - ipip: - description: 'Deprecated: this field is only used for APIv1 backwards - compatibility. Setting this field is not allowed, this field is - for internal use only.' - properties: - enabled: - description: When enabled is true, ipip tunneling will be used - to deliver packets to destinations within this pool. - type: boolean - mode: - description: The IPIP mode. This can be one of "always" or "cross-subnet". A - mode of "always" will also use IPIP tunneling for routing to - destination IP addresses within this pool. A mode of "cross-subnet" - will only use IPIP tunneling when the destination node is on - a different subnet to the originating node. The default value - (if not specified) is "always". - type: string - type: object - ipipMode: - description: Contains configuration for IPIP tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling - is disabled). - type: string - nat-outgoing: - description: 'Deprecated: this field is only used for APIv1 backwards - compatibility. Setting this field is not allowed, this field is - for internal use only.' - type: boolean - natOutgoing: - description: When nat-outgoing is true, packets sent from Calico networked - containers in this pool to destinations outside of this pool will - be masqueraded. - type: boolean - nodeSelector: - description: Allows IPPool to allocate for a specific node by label - selector. - type: string - vxlanMode: - description: Contains configuration for VXLAN tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. VXLAN - tunneling is disabled). - type: string - required: - - cidr - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: ipreservations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPReservation - listKind: IPReservationList - plural: ipreservations - singular: ipreservation - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPReservationSpec contains the specification for an IPReservation - resource. - properties: - reservedCIDRs: - description: ReservedCIDRs is a list of CIDRs and/or IP addresses - that Calico IPAM will exclude from new allocations. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kubecontrollersconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: KubeControllersConfiguration - listKind: KubeControllersConfigurationList - plural: kubecontrollersconfigurations - singular: kubecontrollersconfiguration - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KubeControllersConfigurationSpec contains the values of the - Kubernetes controllers configuration. - properties: - controllers: - description: Controllers enables and configures individual Kubernetes - controllers - properties: - namespace: - description: Namespace enables and configures the namespace controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - node: - description: Node enables and configures the node controller. - Enabled by default, set to nil to disable. - properties: - hostEndpoint: - description: HostEndpoint controls syncing nodes to host endpoints. - Disabled by default, set to nil to disable. - properties: - autoCreate: - description: 'AutoCreate enables automatic creation of - host endpoints for every node. [Default: Disabled]' - type: string - type: object - leakGracePeriod: - description: 'LeakGracePeriod is the period used by the controller - to determine if an IP address has been leaked. Set to 0 - to disable IP garbage collection. [Default: 15m]' - type: string - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - syncLabels: - description: 'SyncLabels controls whether to copy Kubernetes - node labels to Calico nodes. [Default: Enabled]' - type: string - type: object - policy: - description: Policy enables and configures the policy controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - serviceAccount: - description: ServiceAccount enables and configures the service - account controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - workloadEndpoint: - description: WorkloadEndpoint enables and configures the workload - endpoint controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - type: object - debugProfilePort: - description: DebugProfilePort configures the port to serve memory - and cpu profiles on. If not specified, profiling is disabled. - format: int32 - type: integer - etcdV3CompactionPeriod: - description: 'EtcdV3CompactionPeriod is the period between etcdv3 - compaction requests. Set to 0 to disable. [Default: 10m]' - type: string - healthChecks: - description: 'HealthChecks enables or disables support for health - checks [Default: Enabled]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: Info]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. Set to 0 to disable. [Default: 9094]' - type: integer - required: - - controllers - type: object - status: - description: KubeControllersConfigurationStatus represents the status - of the configuration. It's useful for admins to be able to see the actual - config that was applied, which can be modified by environment variables - on the kube-controllers process. - properties: - environmentVars: - additionalProperties: - type: string - description: EnvironmentVars contains the environment variables on - the kube-controllers that influenced the RunningConfig. - type: object - runningConfig: - description: RunningConfig contains the effective config that is running - in the kube-controllers pod, after merging the API resource with - any environment variables. - properties: - controllers: - description: Controllers enables and configures individual Kubernetes - controllers - properties: - namespace: - description: Namespace enables and configures the namespace - controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - node: - description: Node enables and configures the node controller. - Enabled by default, set to nil to disable. - properties: - hostEndpoint: - description: HostEndpoint controls syncing nodes to host - endpoints. Disabled by default, set to nil to disable. - properties: - autoCreate: - description: 'AutoCreate enables automatic creation - of host endpoints for every node. [Default: Disabled]' - type: string - type: object - leakGracePeriod: - description: 'LeakGracePeriod is the period used by the - controller to determine if an IP address has been leaked. - Set to 0 to disable IP garbage collection. [Default: - 15m]' - type: string - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - syncLabels: - description: 'SyncLabels controls whether to copy Kubernetes - node labels to Calico nodes. [Default: Enabled]' - type: string - type: object - policy: - description: Policy enables and configures the policy controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - serviceAccount: - description: ServiceAccount enables and configures the service - account controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - workloadEndpoint: - description: WorkloadEndpoint enables and configures the workload - endpoint controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - type: object - debugProfilePort: - description: DebugProfilePort configures the port to serve memory - and cpu profiles on. If not specified, profiling is disabled. - format: int32 - type: integer - etcdV3CompactionPeriod: - description: 'EtcdV3CompactionPeriod is the period between etcdv3 - compaction requests. Set to 0 to disable. [Default: 10m]' - type: string - healthChecks: - description: 'HealthChecks enables or disables support for health - checks [Default: Enabled]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which - logs are sent to the stdout. [Default: Info]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. Set to 0 to disable. [Default: - 9094]' - type: integer - required: - - controllers - type: object - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: networkpolicies.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: NetworkPolicy - listKind: NetworkPolicyList - plural: networkpolicies - singular: networkpolicy - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - egress: - description: The ordered set of egress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with \"Not\". All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a exact or a prefix match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - ingress: - description: The ordered set of ingress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with \"Not\". All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a exact or a prefix match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel's iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and another selector are defined on the same rule, then - only workload endpoints that are matched by both selectors - will be selected by the rule. \n For NetworkPolicy, an - empty NamespaceSelector implies that the Selector is limited - to selecting only workload endpoints in the same namespace - as the NetworkPolicy. \n For NetworkPolicy, global() - NamespaceSelector implies that the Selector is limited - to selecting only GlobalNetworkSet or HostEndpoint. \n - For GlobalNetworkPolicy, an empty NamespaceSelector implies - the Selector applies to workload endpoints across all - namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label \"my_label\". \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label \"my_label\". - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Nets, NotNets or ServiceAccounts. \n Ports and NotPorts - can only be specified with Services on ingress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - order: - description: Order is an optional field that specifies the order in - which the policy is applied. Policies with higher "order" are applied - after those with lower order. If the order is omitted, it may be - considered to be "infinite" - i.e. the policy will be applied last. Policies - with identical order will be applied in alphanumerical order based - on the Policy "Name". - type: number - selector: - description: "The selector is an expression used to pick pick out - the endpoints that the policy should be applied to. \n Selector - expressions follow this syntax: \n \tlabel == \"string_literal\" - \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" - \ -> not equal; also matches if label is not present \tlabel in - { \"a\", \"b\", \"c\", ... } -> true if the value of label X is - one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", - ... } -> true if the value of label X is not one of \"a\", \"b\", - \"c\" \thas(label_name) -> True if that label is present \t! expr - -> negation of expr \texpr && expr -> Short-circuit and \texpr - || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() - or the empty selector -> matches all endpoints. \n Label names are - allowed to contain alphanumerics, -, _ and /. String literals are - more permissive but they do not support escape characters. \n Examples - (with made-up labels): \n \ttype == \"webserver\" && deployment - == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != - \"dev\" \t! has(label_name)" - type: string - serviceAccountSelector: - description: ServiceAccountSelector is an optional field for an expression - used to select a pod based on service accounts. - type: string - types: - description: "Types indicates whether this policy applies to ingress, - or to egress, or to both. When not explicitly specified (and so - the value on creation is empty or nil), Calico defaults Types according - to what Ingress and Egress are present in the policy. The default - is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including - the case where there are also no Ingress rules) \n - [ PolicyTypeEgress - ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, - PolicyTypeEgress ], if there are both Ingress and Egress rules. - \n When the policy is read back again, Types will always be one - of these values, never empty or nil." - items: - description: PolicyType enumerates the possible values of the PolicySpec - Types field. - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: networksets.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: NetworkSet - listKind: NetworkSetList - plural: networksets - singular: networkset - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NetworkSetSpec contains the specification for a NetworkSet - resource. - properties: - nets: - description: The list of IP networks that belong to this set. - items: - type: string - type: array - type: object - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- ---- -# Source: calico/templates/calico-kube-controllers-rbac.yaml - -# Include a clusterrole for the kube-controllers component, -# and bind it to the calico-kube-controllers serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -rules: - # Nodes are watched to monitor for deletions. - - apiGroups: [""] - resources: - - nodes - verbs: - - watch - - list - - get - # Pods are watched to check for existence as part of IPAM controller. - - apiGroups: [""] - resources: - - pods - verbs: - - get - - list - - watch - # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ipreservations - verbs: - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - - watch - # Pools are watched to maintain a mapping of blocks to IP pools. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - verbs: - - list - - watch - # kube-controllers manages hostendpoints. - - apiGroups: ["crd.projectcalico.org"] - resources: - - hostendpoints - verbs: - - get - - list - - create - - update - - delete - # Needs access to update clusterinformations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - clusterinformations - verbs: - - get - - list - - create - - update - - watch - # KubeControllersConfiguration is where it gets its config - - apiGroups: ["crd.projectcalico.org"] - resources: - - kubecontrollersconfigurations - verbs: - # read its own config - - get - # create a default if none exists - - create - # update status - - update - # watch for changes - - watch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-kube-controllers -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-kube-controllers -subjects: -- kind: ServiceAccount - name: calico-kube-controllers - namespace: kube-system ---- - ---- -# Source: calico/templates/calico-node-rbac.yaml -# Include a clusterrole for the calico-node DaemonSet, -# and bind it to the calico-node serviceaccount. -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: calico-node -rules: - # The CNI plugin needs to get pods, nodes, and namespaces. - - apiGroups: [""] - resources: - - pods - - nodes - - namespaces - verbs: - - get - # EndpointSlices are used for Service-based network policy rule - # enforcement. - - apiGroups: ["discovery.k8s.io"] - resources: - - endpointslices - verbs: - - watch - - list - - apiGroups: [""] - resources: - - endpoints - - services - verbs: - # Used to discover service IPs for advertisement. - - watch - - list - # Used to discover Typhas. - - get - # Pod CIDR auto-detection on kubeadm needs access to config maps. - - apiGroups: [""] - resources: - - configmaps - verbs: - - get - - apiGroups: [""] - resources: - - nodes/status - verbs: - # Needed for clearing NodeNetworkUnavailable flag. - - patch - # Calico stores some configuration information in node annotations. - - update - # Watch for changes to Kubernetes NetworkPolicies. - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - # Used by Calico for policy information. - - apiGroups: [""] - resources: - - pods - - namespaces - - serviceaccounts - verbs: - - list - - watch - # The CNI plugin patches pods/status. - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - # Used for creating service account tokens to be used by the CNI plugin - - apiGroups: [""] - resources: - - serviceaccounts/token - resourceNames: - - calico-node - verbs: - - create - # Calico monitors various CRDs for config. - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - ipreservations - - ipamblocks - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - networksets - - clusterinformations - - hostendpoints - - blockaffinities - - caliconodestatuses - verbs: - - get - - list - - watch - # Calico must create and update some CRDs on startup. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - - felixconfigurations - - clusterinformations - verbs: - - create - - update - # Calico must update some CRDs. - - apiGroups: [ "crd.projectcalico.org" ] - resources: - - caliconodestatuses - verbs: - - update - # Calico stores some configuration information on the node. - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - watch - # These permissions are only required for upgrade from v2.6, and can - # be removed after upgrade or on fresh installations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - bgpconfigurations - - bgppeers - verbs: - - create - - update - # These permissions are required for Calico CNI to perform IPAM allocations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - - apiGroups: ["crd.projectcalico.org"] - resources: - - ipamconfigs - verbs: - - get - # Block affinities must also be watchable by confd for route aggregation. - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - verbs: - - watch - # The Calico IPAM migration needs to get daemonsets. These permissions can be - # removed if not upgrading from an installation using host-local IPAM. - - apiGroups: ["apps"] - resources: - - daemonsets - verbs: - - get - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: calico-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node -subjects: -- kind: ServiceAccount - name: calico-node - namespace: kube-system - ---- -# Source: calico/templates/calico-node.yaml -# This manifest installs the calico-node container, as well -# as the CNI plugins and network config on -# each master and worker node in a Kubernetes cluster. -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: calico-node - namespace: kube-system - labels: - k8s-app: calico-node -spec: - selector: - matchLabels: - k8s-app: calico-node - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - template: - metadata: - labels: - k8s-app: calico-node - spec: - nodeSelector: - kubernetes.io/os: linux - hostNetwork: true - tolerations: - # Make sure calico-node gets scheduled on all nodes. - - effect: NoSchedule - operator: Exists - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - serviceAccountName: calico-node - # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force - # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. - terminationGracePeriodSeconds: 0 - priorityClassName: system-node-critical - initContainers: - # This container performs upgrade from host-local IPAM to calico-ipam. - # It can be deleted if this is a fresh installation, or if you have already - # upgraded to use calico-ipam. - - name: upgrade-ipam - image: docker.io/calico/cni:v3.23.2 - command: ["/opt/cni/bin/calico-ipam", "-upgrade"] - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - volumeMounts: - - mountPath: /var/lib/cni/networks - name: host-local-net-dir - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - securityContext: - privileged: true - # This container installs the CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: docker.io/calico/cni:v3.23.2 - command: ["/opt/cni/bin/install"] - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # CNI MTU Config variable - - name: CNI_MTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Prevents the container from sleeping forever. - - name: SLEEP - value: "false" - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - securityContext: - privileged: true - # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes - # to communicate with Felix over the Policy Sync API. - - name: flexvol-driver - image: docker.io/calico/pod2daemon-flexvol:v3.23.2 - volumeMounts: - - name: flexvol-driver-host - mountPath: /host/driver - securityContext: - privileged: true - containers: - # Runs calico-node container on each Kubernetes node. This - # container programs network policy and routes on each - # host. - - name: calico-node - image: docker.io/calico/node:v3.23.2 - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # Set based on the k8s node name. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # Choose the backend to use. - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s,bgp" - # Auto-detect the BGP IP address. - - name: NODEIP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: IP_AUTODETECTION_METHOD - value: "can-reach=$(NODEIP)" - # Auto-detect the BGP IP address. - - name: IP - value: "autodetect" - # Enable IPIP - - name: CALICO_IPV4POOL_IPIP - value: "Always" - # Enable or Disable VXLAN on the default IP pool. - - name: CALICO_IPV4POOL_VXLAN - value: "Never" - # Set MTU for tunnel device used if ipip is enabled - - name: FELIX_IPINIPMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Set MTU for the VXLAN tunnel device. - - name: FELIX_VXLANMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Set MTU for the Wireguard tunnel device. - - name: FELIX_WIREGUARDMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # The default IPv4 pool to create on startup if none exists. Pod IPs will be - # chosen from this range. Changing this value after installation will have - # no effect. This should fall within `--cluster-cidr`. - - name: CALICO_IPV4POOL_CIDR - value: '${POD_CIDRS}' - - name: CALICO_IPV4POOL_BLOCK_SIZE - value: "24" - # Disable file logging so `kubectl logs` works. - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPv6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - - name: FELIX_HEALTHENABLED - value: "true" - securityContext: - privileged: true - resources: - requests: - cpu: 250m - livenessProbe: - exec: - command: - - /bin/calico-node - - -felix-live - - -bird-live - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - readinessProbe: - exec: - command: - - /bin/calico-node - - -felix-ready - - -bird-ready - periodSeconds: 10 - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /run/xtables.lock - name: xtables-lock - readOnly: false - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - - name: policysync - mountPath: /var/run/nodeagent - # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the - # parent directory. - - name: sysfs - mountPath: /sys/fs/ - # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host. - # If the host is known to mount that filesystem already then Bidirectional can be omitted. - mountPropagation: Bidirectional - - name: cni-log-dir - mountPath: /var/log/calico/cni - readOnly: true - volumes: - # Used by calico-node. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - - name: sysfs - hostPath: - path: /sys/fs/ - type: DirectoryOrCreate - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d - # Used to access CNI logs. - - name: cni-log-dir - hostPath: - path: /var/log/calico/cni - # Mount in the directory for host-local IPAM allocations. This is - # used when upgrading from host-local to calico-ipam, and can be removed - # if not using the upgrade-ipam init container. - - name: host-local-net-dir - hostPath: - path: /var/lib/cni/networks - # Used to create per-pod Unix Domain Sockets - - name: policysync - hostPath: - type: DirectoryOrCreate - path: /var/run/nodeagent - # Used to install Flex Volume Driver - - name: flexvol-driver-host - hostPath: - type: DirectoryOrCreate - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-node - namespace: kube-system - ---- -# Source: calico/templates/calico-kube-controllers.yaml -# See https://github.com/projectcalico/kube-controllers -apiVersion: apps/v1 -kind: Deployment -metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers -spec: - # The controllers can only have a single active instance. - replicas: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers - strategy: - type: Recreate - template: - metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers - spec: - nodeSelector: - kubernetes.io/os: linux - tolerations: - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - effect: NoSchedule - serviceAccountName: calico-kube-controllers - priorityClassName: system-cluster-critical - containers: - - name: calico-kube-controllers - image: docker.io/calico/kube-controllers:v3.23.2 - env: - # Choose which controllers to run. - - name: ENABLED_CONTROLLERS - value: node - - name: DATASTORE_TYPE - value: kubernetes - livenessProbe: - exec: - command: - - /usr/bin/check-status - - -l - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - timeoutSeconds: 10 - readinessProbe: - exec: - command: - - /usr/bin/check-status - - -r - periodSeconds: 10 - ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: calico-kube-controllers - namespace: kube-system - ---- - -# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict - -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers -spec: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers - ---- -# Source: calico/templates/calico-etcd-secrets.yaml - ---- -# Source: calico/templates/calico-typha.yaml - ---- -# Source: calico/templates/configure-canal.yaml - - diff --git a/test/e2e/data/infrastructure-kubekey/v1beta1/bases/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-kubekey/v1beta1/bases/cluster-with-kcp.yaml deleted file mode 100644 index adce9e29..00000000 --- a/test/e2e/data/infrastructure-kubekey/v1beta1/bases/cluster-with-kcp.yaml +++ /dev/null @@ -1,159 +0,0 @@ ---- -# KKCluster object referenced by the Cluster object -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKCluster -metadata: - name: '${CLUSTER_NAME}' -spec: - distribution: kubernetes - component: - zone: '${KKZONE}' - nodes: - auth: - user: '${USER_NAME}' - password: '${PASSWORD}' - instances: '${INSTANCES}' - controlPlaneLoadBalancer: - host: '${CONTROL_PLANE_ENDPOINT_IP}' ---- -# Cluster object with -# - Reference to the KubeadmControlPlane object -# - the label cni=${CLUSTER_NAME}-crs-0, so the cluster can be selected by the ClusterResourceSet. -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: '${CLUSTER_NAME}' - labels: - cni: "${CLUSTER_NAME}-crs-0" -spec: - clusterNetwork: - services: - cidrBlocks: ['${SERVICE_CIDRS}'] - pods: - cidrBlocks: ['${POD_CIDRS}'] - serviceDomain: '${SERVICE_DOMAIN}' - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: KKCluster - name: '${CLUSTER_NAME}' - controlPlaneRef: - kind: KubeadmControlPlane - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - name: "${CLUSTER_NAME}-control-plane" ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKMachineTemplate -metadata: - name: "${CLUSTER_NAME}-control-plane" -spec: - template: - spec: - containerManager: - type: ${CONTAINER_MANAGER_TYPE} - roles: - - control-plane - repository: - iso: "none" ---- -# KubeadmControlPlane referenced by the Cluster object with -# - the label kcp-adoption.step2, because it should be created in the second step of the kcp-adoption test. -kind: KubeadmControlPlane -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -metadata: - name: "${CLUSTER_NAME}-control-plane" - labels: - kcp-adoption.step2: "" -spec: - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - machineTemplate: - infrastructureRef: - kind: KKMachineTemplate - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - name: "${CLUSTER_NAME}-control-plane" - kubeadmConfigSpec: - clusterConfiguration: - controllerManager: - extraArgs: {enable-hostpath-provisioner: 'true'} - imageRepository: ${IMAGE_REPOSITORY} - apiServer: - # host.docker.internal is required by kubetest when running on macOS because of the way ports are proxied. - certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal] - files: - - content: | - apiVersion: v1 - kind: Pod - metadata: - creationTimestamp: null - name: kube-vip - namespace: kube-system - spec: - containers: - - args: - - manager - env: - - name: address - value: ${CONTROL_PLANE_ENDPOINT_IP} - - name: vip_interface - value: ${VIP_NETWORK_INTERFACE=""} - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: lb_enable - value: "true" - - name: lb_port - value: "6443" - image: ghcr.io/kube-vip/kube-vip:v0.5.0 - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - hostNetwork: true - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - volumes: - - hostPath: - path: /etc/kubernetes/admin.conf - type: FileOrCreate - name: kubeconfig - status: {} - owner: root:root - path: /etc/kubernetes/manifests/kube-vip.yaml - initConfiguration: - nodeRegistration: - criSocket: ${CRI_SOCKET} - kubeletExtraArgs: - eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' - joinConfiguration: - nodeRegistration: - criSocket: ${CRI_SOCKET} - kubeletExtraArgs: - eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' - version: "${KUBERNETES_VERSION}" diff --git a/test/e2e/data/infrastructure-kubekey/v1beta1/bases/crs.yaml b/test/e2e/data/infrastructure-kubekey/v1beta1/bases/crs.yaml deleted file mode 100644 index b1b61237..00000000 --- a/test/e2e/data/infrastructure-kubekey/v1beta1/bases/crs.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -# ConfigMap object referenced by the ClusterResourceSet object and with -# the CNI resource defined in the test config file -apiVersion: v1 -kind: ConfigMap -metadata: - name: "cni-${CLUSTER_NAME}-crs-0" -data: ${CNI_RESOURCES} -binaryData: ---- -# ClusterResourceSet object with -# a selector that targets all the Cluster with label cni=${CLUSTER_NAME}-crs-0 -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: "${CLUSTER_NAME}-crs-0" -spec: - strategy: ApplyOnce - clusterSelector: - matchLabels: - cni: "${CLUSTER_NAME}-crs-0" - resources: - - name: "cni-${CLUSTER_NAME}-crs-0" - kind: ConfigMap diff --git a/test/e2e/data/infrastructure-kubekey/v1beta1/bases/md.yaml b/test/e2e/data/infrastructure-kubekey/v1beta1/bases/md.yaml deleted file mode 100644 index 364d06d9..00000000 --- a/test/e2e/data/infrastructure-kubekey/v1beta1/bases/md.yaml +++ /dev/null @@ -1,54 +0,0 @@ ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKMachineTemplate -metadata: - name: '${CLUSTER_NAME}-md-0' -spec: - template: - spec: - containerManager: - type: ${CONTAINER_MANAGER_TYPE} - roles: - - control-plane - - worker - repository: - iso: "none" ---- -# KubeadmConfigTemplate referenced by the MachineDeployment -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: "${CLUSTER_NAME}-md-0" -spec: - template: - spec: - joinConfiguration: - nodeRegistration: - criSocket: ${CRI_SOCKET} - kubeletExtraArgs: - eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' ---- -# MachineDeployment object -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: "${CLUSTER_NAME}-md-0" -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: - template: - spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" - bootstrap: - configRef: - name: "${CLUSTER_NAME}-md-0" - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - infrastructureRef: - name: "${CLUSTER_NAME}-md-0" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: KKMachineTemplate - failureDomain: fd4 diff --git a/test/e2e/data/infrastructure-kubekey/v1beta1/cluster-template/kustomization.yaml b/test/e2e/data/infrastructure-kubekey/v1beta1/cluster-template/kustomization.yaml deleted file mode 100644 index adb5919c..00000000 --- a/test/e2e/data/infrastructure-kubekey/v1beta1/cluster-template/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -bases: -- ../bases/cluster-with-kcp.yaml -- ../bases/md.yaml -- ../bases/crs.yaml - diff --git a/test/e2e/data/k3s/v1beta1/bases/cluster-with-kcp.yaml b/test/e2e/data/k3s/v1beta1/bases/cluster-with-kcp.yaml deleted file mode 100644 index 8dba328b..00000000 --- a/test/e2e/data/k3s/v1beta1/bases/cluster-with-kcp.yaml +++ /dev/null @@ -1,196 +0,0 @@ ---- -# KKCluster object referenced by the Cluster object -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKCluster -metadata: - name: '${CLUSTER_NAME}' -spec: - distribution: k3s - component: - zone: '${KKZONE}' - nodes: - auth: - user: '${USER_NAME}' - password: '${PASSWORD}' - instances: '${INSTANCES}' - controlPlaneLoadBalancer: - host: '${CONTROL_PLANE_ENDPOINT_IP}' ---- -# Cluster object with -# - Reference to the K3sControlPlane object -# - the label cni=${CLUSTER_NAME}-crs-0, so the cluster can be selected by the ClusterResourceSet. -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: '${CLUSTER_NAME}' - labels: - cni: "${CLUSTER_NAME}-crs-0" -spec: - clusterNetwork: - services: - cidrBlocks: ['${SERVICE_CIDRS}'] - pods: - cidrBlocks: ['${POD_CIDRS}'] - serviceDomain: '${SERVICE_DOMAIN}' - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: KKCluster - name: '${CLUSTER_NAME}' - controlPlaneRef: - kind: K3sControlPlane - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - name: "${CLUSTER_NAME}-control-plane" ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKMachineTemplate -metadata: - name: "${CLUSTER_NAME}-control-plane" -spec: - template: - spec: - roles: - - control-plane ---- -kind: K3sControlPlane -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -metadata: - name: "${CLUSTER_NAME}-control-plane" -spec: - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - machineTemplate: - infrastructureRef: - kind: KKMachineTemplate - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - name: "${CLUSTER_NAME}-control-plane" - k3sConfigSpec: - serverConfiguration: - listener: - tlsSan: ${CONTROL_PLANE_ENDPOINT_IP} - networking: - flannelBackend: none - kubernetesComponents: - disable: "servicelb,traefik,metrics-server,local-storage" - disableNetworkPolicy: true - files: - - content: | - apiVersion: v1 - kind: ServiceAccount - metadata: - name: kube-vip - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role - rules: - - apiGroups: [""] - resources: ["services", "services/status", "nodes", "endpoints"] - verbs: ["list","get","watch", "update"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: system:kube-vip-binding - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-vip-role - subjects: - - kind: ServiceAccount - name: kube-vip - namespace: kube-system - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.5.0 - name: kube-vip-ds - namespace: kube-system - spec: - selector: - matchLabels: - app.kubernetes.io/name: kube-vip-ds - template: - metadata: - creationTimestamp: null - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.5.0 - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - containers: - - args: - - manager - env: - - name: address - value: ${CONTROL_PLANE_ENDPOINT_IP} - - name: vip_interface - value: ${VIP_NETWORK_INTERFACE=""} - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: lb_enable - value: "true" - - name: lb_port - value: "6443" - - name: lb_fwdmethod - value: local - - name: prometheus_server - value: :2112 - image: ghcr.io/kube-vip/kube-vip:v0.5.0 - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - hostNetwork: true - serviceAccountName: kube-vip - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - updateStrategy: {} - owner: root:root - path: /var/lib/rancher/k3s/server/manifests/kube-vip.yaml - version: "${KUBERNETES_VERSION}" diff --git a/test/e2e/data/k3s/v1beta1/bases/crs.yaml b/test/e2e/data/k3s/v1beta1/bases/crs.yaml deleted file mode 100644 index b62f66d3..00000000 --- a/test/e2e/data/k3s/v1beta1/bases/crs.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -# ConfigMap object referenced by the ClusterResourceSet object and with -# the CNI resource defined in the test config file -apiVersion: v1 -kind: ConfigMap -metadata: - name: "cni-${CLUSTER_NAME}-crs-0" -data: ${CNI_RESOURCES} ---- -# ClusterResourceSet object with -# a selector that targets all the Cluster with label cni=${CLUSTER_NAME}-crs-0 -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: "${CLUSTER_NAME}-crs-0" -spec: - strategy: ApplyOnce - clusterSelector: - matchLabels: - cni: "${CLUSTER_NAME}-crs-0" - resources: - - name: "cni-${CLUSTER_NAME}-crs-0" - kind: ConfigMap diff --git a/test/e2e/data/k3s/v1beta1/bases/md.yaml b/test/e2e/data/k3s/v1beta1/bases/md.yaml deleted file mode 100644 index cb7fc2c6..00000000 --- a/test/e2e/data/k3s/v1beta1/bases/md.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: KKMachineTemplate -metadata: - name: '${CLUSTER_NAME}-md-0' -spec: - template: - spec: - roles: - - control-plane - - worker ---- -# KubeadmConfigTemplate referenced by the MachineDeployment -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: K3sConfigTemplate -metadata: - name: "${CLUSTER_NAME}-md-0" ---- -# MachineDeployment object -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: "${CLUSTER_NAME}-md-0" -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: - template: - spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" - bootstrap: - configRef: - name: "${CLUSTER_NAME}-md-0" - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: K3sConfigTemplate - infrastructureRef: - name: "${CLUSTER_NAME}-md-0" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: KKMachineTemplate - failureDomain: fd4 diff --git a/test/e2e/data/k3s/v1beta1/cluster-template/kustomization.yaml b/test/e2e/data/k3s/v1beta1/cluster-template/kustomization.yaml deleted file mode 100644 index adb5919c..00000000 --- a/test/e2e/data/k3s/v1beta1/cluster-template/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -bases: -- ../bases/cluster-with-kcp.yaml -- ../bases/md.yaml -- ../bases/crs.yaml - diff --git a/test/e2e/data/shared/v1beta1/metadata.yaml b/test/e2e/data/shared/v1beta1/metadata.yaml deleted file mode 100644 index d240815a..00000000 --- a/test/e2e/data/shared/v1beta1/metadata.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# maps release series of major.minor to cluster-api contract version -# the contract version may change between minor or major versions, but *not* -# between patch versions. -# -# update this file only when a new major or minor version is released -apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 -releaseSeries: - - major: 0 - minor: 4 - contract: v1alpha4 - - major: 0 - minor: 3 - contract: v1alpha3 - - major: 1 - minor: 0 - contract: v1beta1 - - major: 1 - minor: 1 - contract: v1beta1 - - major: 1 - minor: 2 - contract: v1beta1 diff --git a/test/e2e/data/shared/v1beta1_provider/metadata.yaml b/test/e2e/data/shared/v1beta1_provider/metadata.yaml deleted file mode 100644 index e0ec3ba5..00000000 --- a/test/e2e/data/shared/v1beta1_provider/metadata.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# maps release series of major.minor to cluster-api contract version -# the contract version may change between minor or major versions, but *not* -# between patch versions. -# -# update this file only when a new major or minor version is released -apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 -releaseSeries: - - major: 3 - minor: 0 - contract: v1beta1 diff --git a/test/e2e/doc.go b/test/e2e/doc.go deleted file mode 100644 index 5761c922..00000000 --- a/test/e2e/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package e2e implements end to end testing. -package e2e diff --git a/test/e2e/framework/alltypes_helpers.go b/test/e2e/framework/alltypes_helpers.go deleted file mode 100644 index 3703f822..00000000 --- a/test/e2e/framework/alltypes_helpers.go +++ /dev/null @@ -1,184 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path" - "path/filepath" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - apimeta "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/yaml" -) - -// GetCAPIResourcesInput is the input for GetCAPIResources. -type GetCAPIResourcesInput struct { - Lister Lister - Namespace string -} - -// GetCAPIResources reads all the CAPI resources in a namespace. -// This list includes all the types belonging to CAPI providers. -func GetCAPIResources(ctx context.Context, input GetCAPIResourcesInput) []*unstructured.Unstructured { - Expect(ctx).NotTo(BeNil(), "ctx is required for GetCAPIResources") - Expect(input.Lister).NotTo(BeNil(), "input.Deleter is required for GetCAPIResources") - Expect(input.Namespace).NotTo(BeEmpty(), "input.Namespace is required for GetCAPIResources") - - types := getClusterAPITypes(ctx, input.Lister) - - objList := []*unstructured.Unstructured{} - for i := range types { - typeMeta := types[i] - typeList := new(unstructured.UnstructuredList) - typeList.SetAPIVersion(typeMeta.APIVersion) - typeList.SetKind(typeMeta.Kind) - - if err := input.Lister.List(ctx, typeList, client.InNamespace(input.Namespace)); err != nil { - if apierrors.IsNotFound(err) { - continue - } - Fail(fmt.Sprintf("failed to list %q resources: %v", typeList.GroupVersionKind(), err)) - } - for i := range typeList.Items { - obj := typeList.Items[i] - objList = append(objList, &obj) - } - } - - return objList -} - -// getClusterAPITypes returns the list of TypeMeta to be considered for the move discovery phase. -// This list includes all the types belonging to CAPI providers. -func getClusterAPITypes(ctx context.Context, lister Lister) []metav1.TypeMeta { - discoveredTypes := []metav1.TypeMeta{} - - crdList := &apiextensionsv1.CustomResourceDefinitionList{} - Eventually(func() error { - return lister.List(ctx, crdList, capiProviderOptions()...) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "failed to list CRDs for CAPI providers") - - for _, crd := range crdList.Items { - for _, version := range crd.Spec.Versions { - if !version.Storage { - continue - } - - discoveredTypes = append(discoveredTypes, metav1.TypeMeta{ - Kind: crd.Spec.Names.Kind, - APIVersion: metav1.GroupVersion{ - Group: crd.Spec.Group, - Version: version.Name, - }.String(), - }) - } - } - return discoveredTypes -} - -// DumpAllResourcesInput is the input for DumpAllResources. -type DumpAllResourcesInput struct { - Lister Lister - Namespace string - LogPath string -} - -// DumpAllResources dumps Cluster API related resources to YAML -// This dump includes all the types belonging to CAPI providers. -func DumpAllResources(ctx context.Context, input DumpAllResourcesInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for DumpAllResources") - Expect(input.Lister).NotTo(BeNil(), "input.Deleter is required for DumpAllResources") - Expect(input.Namespace).NotTo(BeEmpty(), "input.Namespace is required for DumpAllResources") - - resources := GetCAPIResources(ctx, GetCAPIResourcesInput{ - Lister: input.Lister, - Namespace: input.Namespace, - }) - - for i := range resources { - r := resources[i] - dumpObject(r, input.LogPath) - } -} - -func dumpObject(resource runtime.Object, logPath string) { - resourceYAML, err := yaml.Marshal(resource) - Expect(err).ToNot(HaveOccurred(), "Failed to marshal %s", resource.GetObjectKind().GroupVersionKind().String()) - - metaObj, err := apimeta.Accessor(resource) - Expect(err).ToNot(HaveOccurred(), "Failed to get accessor for %s", resource.GetObjectKind().GroupVersionKind().String()) - - kind := resource.GetObjectKind().GroupVersionKind().Kind - namespace := metaObj.GetNamespace() - name := metaObj.GetName() - - resourceFilePath := filepath.Clean(path.Join(logPath, namespace, kind, name+".yaml")) - Expect(os.MkdirAll(filepath.Dir(resourceFilePath), 0750)).To(Succeed(), "Failed to create folder %s", filepath.Dir(resourceFilePath)) - - f, err := os.OpenFile(resourceFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) - Expect(err).ToNot(HaveOccurred(), "Failed to open %s", resourceFilePath) - defer f.Close() - - Expect(os.WriteFile(f.Name(), resourceYAML, 0600)).To(Succeed(), "Failed to write %s", resourceFilePath) -} - -// capiProviderOptions returns a set of ListOptions that allows to identify all the objects belonging to Cluster API providers. -func capiProviderOptions() []client.ListOption { - return []client.ListOption{ - client.HasLabels{clusterv1.ProviderLabelName}, - } -} - -// CreateRelatedResourcesInput is the input type for CreateRelatedResources. -type CreateRelatedResourcesInput struct { - Creator Creator - RelatedResources []client.Object -} - -// CreateRelatedResources is used to create runtime.Objects. -func CreateRelatedResources(ctx context.Context, input CreateRelatedResourcesInput, intervals ...interface{}) { - By("creating related resources") - for i := range input.RelatedResources { - obj := input.RelatedResources[i] - Byf("creating a/an %s resource", obj.GetObjectKind().GroupVersionKind()) - Eventually(func() error { - return input.Creator.Create(ctx, obj) - }, intervals...).Should(Succeed(), "failed to create %s", obj.GetObjectKind().GroupVersionKind()) - } -} - -// PrettyPrint returns a formatted JSON version of the object given. -func PrettyPrint(v interface{}) string { - b, err := json.MarshalIndent(v, "", " ") - if err != nil { - return err.Error() - } - return string(b) -} diff --git a/test/e2e/framework/bootstrap/interfaces.go b/test/e2e/framework/bootstrap/interfaces.go deleted file mode 100644 index df86a470..00000000 --- a/test/e2e/framework/bootstrap/interfaces.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package bootstrap implements bootstrap functionality for e2e testing. -package bootstrap - -import "context" - -// ClusterProvider defines the behavior of a type that is responsible for provisioning and managing a Kubernetes cluster. -type ClusterProvider interface { - // Create a Kubernetes cluster. - // Generally to be used in the BeforeSuite function to create a Kubernetes cluster to be shared between tests. - Create(context.Context) - - // GetKubeconfigPath returns the path to the kubeconfig file to be used to access the Kubernetes cluster. - GetKubeconfigPath() string - - // Dispose will completely clean up the provisioned cluster. - // This should be implemented as a synchronous function. - // Generally to be used in the AfterSuite function if a Kubernetes cluster is shared between tests. - // Should try to clean everything up and report any dangling artifacts that needs manual intervention. - Dispose(context.Context) -} diff --git a/test/e2e/framework/bootstrap/kind_provider.go b/test/e2e/framework/bootstrap/kind_provider.go deleted file mode 100644 index 5f4551aa..00000000 --- a/test/e2e/framework/bootstrap/kind_provider.go +++ /dev/null @@ -1,204 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package bootstrap - -import ( - "context" - "fmt" - "os" - - . "github.com/onsi/gomega" - "github.com/pkg/errors" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - kindv1 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4" - kind "sigs.k8s.io/kind/pkg/cluster" - "sigs.k8s.io/kind/pkg/cmd" - "sigs.k8s.io/kind/pkg/exec" - - "github.com/kubesphere/kubekey/v3/test/e2e/framework/internal/log" -) - -const ( - // DefaultNodeImageRepository is the default node image repository to be used for testing. - DefaultNodeImageRepository = "kindest/node" - - // DefaultNodeImageVersion is the default Kubernetes version to be used for creating a kind cluster. - DefaultNodeImageVersion = "v1.25.0" -) - -// KindClusterOption is a NewKindClusterProvider option. -type KindClusterOption interface { - apply(*KindClusterProvider) -} - -type kindClusterOptionAdapter func(*KindClusterProvider) - -func (adapter kindClusterOptionAdapter) apply(kindClusterProvider *KindClusterProvider) { - adapter(kindClusterProvider) -} - -// WithNodeImage implements a New Option that instruct the kindClusterProvider to use a specific node image / Kubernetes version. -func WithNodeImage(image string) KindClusterOption { - return kindClusterOptionAdapter(func(k *KindClusterProvider) { - k.nodeImage = image - }) -} - -// WithDockerSockMount implements a New Option that instruct the kindClusterProvider to mount /var/run/docker.sock into -// the new kind cluster. -func WithDockerSockMount() KindClusterOption { - return kindClusterOptionAdapter(func(k *KindClusterProvider) { - k.withDockerSock = true - }) -} - -// WithIPv6Family implements a New Option that instruct the kindClusterProvider to set the IPFamily to IPv6 in -// the new kind cluster. -func WithIPv6Family() KindClusterOption { - return kindClusterOptionAdapter(func(k *KindClusterProvider) { - k.ipFamily = clusterv1.IPv6IPFamily - }) -} - -// LogFolder implements a New Option that instruct the kindClusterProvider to dump bootstrap logs in a folder in case of errors. -func LogFolder(path string) KindClusterOption { - return kindClusterOptionAdapter(func(k *KindClusterProvider) { - k.logFolder = path - }) -} - -// NewKindClusterProvider returns a ClusterProvider that can create a kind cluster. -func NewKindClusterProvider(name string, options ...KindClusterOption) *KindClusterProvider { - Expect(name).ToNot(BeEmpty(), "name is required for NewKindClusterProvider") - - clusterProvider := &KindClusterProvider{ - name: name, - } - for _, option := range options { - option.apply(clusterProvider) - } - return clusterProvider -} - -// KindClusterProvider implements a ClusterProvider that can create a kind cluster. -type KindClusterProvider struct { - name string - withDockerSock bool - kubeconfigPath string - nodeImage string - ipFamily clusterv1.ClusterIPFamily - logFolder string -} - -// Create a Kubernetes cluster using kind. -func (k *KindClusterProvider) Create(ctx context.Context) { - Expect(ctx).NotTo(BeNil(), "ctx is required for Create") - - // Sets the kubeconfig path to a temp file. - // NB. the ClusterProvider is responsible for the cleanup of this file - f, err := os.CreateTemp("", "e2e-kind") - Expect(err).ToNot(HaveOccurred(), "Failed to create kubeconfig file for the kind cluster %q", k.name) - k.kubeconfigPath = f.Name() - - // Creates the kind cluster - k.createKindCluster() -} - -// createKindCluster calls the kind library taking care of passing options for: -// - use a dedicated kubeconfig file (test should not alter the user environment) -// - if required, mount /var/run/docker.sock. -func (k *KindClusterProvider) createKindCluster() { - kindCreateOptions := []kind.CreateOption{ - kind.CreateWithKubeconfigPath(k.kubeconfigPath), - } - - cfg := &kindv1.Cluster{ - TypeMeta: kindv1.TypeMeta{ - APIVersion: "kind.x-k8s.io/v1alpha4", - Kind: "Cluster", - }, - } - - if k.ipFamily == clusterv1.IPv6IPFamily { - cfg.Networking.IPFamily = kindv1.IPv6Family - } - kindv1.SetDefaultsCluster(cfg) - - if k.withDockerSock { - setDockerSockConfig(cfg) - } - - kindCreateOptions = append(kindCreateOptions, kind.CreateWithV1Alpha4Config(cfg)) - - nodeImage := fmt.Sprintf("%s:%s", DefaultNodeImageRepository, DefaultNodeImageVersion) - if k.nodeImage != "" { - nodeImage = k.nodeImage - } - kindCreateOptions = append(kindCreateOptions, kind.CreateWithNodeImage(nodeImage)) - kindCreateOptions = append(kindCreateOptions, kind.CreateWithRetain(true)) - - provider := kind.NewProvider(kind.ProviderWithLogger(cmd.NewLogger())) - err := provider.Create(k.name, kindCreateOptions...) - if err != nil { - // if requested, dump kind logs - if k.logFolder != "" { - if err := provider.CollectLogs(k.name, k.logFolder); err != nil { - log.Logf("Failed to collect logs from kind: %v", err) - } - } - - errStr := fmt.Sprintf("Failed to create kind cluster %q: %v", k.name, err) - // Extract the details of the RunError, if the cluster creation was triggered by a RunError. - var runErr *exec.RunError - if errors.As(err, &runErr) { - errStr += "\n" + string(runErr.Output) - } - Expect(err).ToNot(HaveOccurred(), errStr) - } -} - -// setDockerSockConfig returns a kind config for mounting /var/run/docker.sock into the kind node. -func setDockerSockConfig(cfg *kindv1.Cluster) { - cfg.Nodes = []kindv1.Node{ - { - Role: kindv1.ControlPlaneRole, - ExtraMounts: []kindv1.Mount{ - { - HostPath: "/var/run/docker.sock", - ContainerPath: "/var/run/docker.sock", - }, - }, - }, - } -} - -// GetKubeconfigPath returns the path to the kubeconfig file for the cluster. -func (k *KindClusterProvider) GetKubeconfigPath() string { - return k.kubeconfigPath -} - -// Dispose the kind cluster and its kubeconfig file. -func (k *KindClusterProvider) Dispose(ctx context.Context) { - Expect(ctx).NotTo(BeNil(), "ctx is required for Dispose") - - if err := kind.NewProvider().Delete(k.name, k.kubeconfigPath); err != nil { - log.Logf("Deleting the kind cluster %q failed. You may need to remove this by hand.", k.name) - } - if err := os.Remove(k.kubeconfigPath); err != nil { - log.Logf("Deleting the kubeconfig file %q file. You may need to remove this by hand.", k.kubeconfigPath) - } -} diff --git a/test/e2e/framework/bootstrap/kind_util.go b/test/e2e/framework/bootstrap/kind_util.go deleted file mode 100644 index 1e5efabf..00000000 --- a/test/e2e/framework/bootstrap/kind_util.go +++ /dev/null @@ -1,197 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package bootstrap - -import ( - "context" - "fmt" - "os" - "path/filepath" - - . "github.com/onsi/gomega" - "github.com/pkg/errors" - "sigs.k8s.io/cluster-api/test/infrastructure/container" - kind "sigs.k8s.io/kind/pkg/cluster" - kindnodes "sigs.k8s.io/kind/pkg/cluster/nodes" - kindnodesutils "sigs.k8s.io/kind/pkg/cluster/nodeutils" - - "github.com/kubesphere/kubekey/v3/test/e2e/framework/clusterctl" - "github.com/kubesphere/kubekey/v3/test/e2e/framework/internal/log" -) - -// CreateKindBootstrapClusterAndLoadImagesInput is the input for CreateKindBootstrapClusterAndLoadImages. -type CreateKindBootstrapClusterAndLoadImagesInput struct { - // Name of the cluster. - Name string - - // KubernetesVersion of the cluster. - KubernetesVersion string - - // RequiresDockerSock defines if the cluster requires the docker sock. - RequiresDockerSock bool - - // Images to be loaded in the cluster. - Images []clusterctl.ContainerImage - - // IPFamily is either ipv4 or ipv6. Default is ipv4. - IPFamily string - - // LogFolder where to dump logs in case of errors - LogFolder string -} - -// CreateKindBootstrapClusterAndLoadImages returns a new Kubernetes cluster with pre-loaded images. -func CreateKindBootstrapClusterAndLoadImages(ctx context.Context, input CreateKindBootstrapClusterAndLoadImagesInput) ClusterProvider { - Expect(ctx).NotTo(BeNil(), "ctx is required for CreateKindBootstrapClusterAndLoadImages") - Expect(input.Name).ToNot(BeEmpty(), "Invalid argument. Name can't be empty when calling CreateKindBootstrapClusterAndLoadImages") - - log.Logf("Creating a kind cluster with name %q", input.Name) - - options := []KindClusterOption{} - if input.KubernetesVersion != "" { - options = append(options, WithNodeImage(fmt.Sprintf("%s:%s", DefaultNodeImageRepository, input.KubernetesVersion))) - } - if input.RequiresDockerSock { - options = append(options, WithDockerSockMount()) - } - if input.IPFamily == "IPv6" { - options = append(options, WithIPv6Family()) - } - if input.LogFolder != "" { - options = append(options, LogFolder(input.LogFolder)) - } - - clusterProvider := NewKindClusterProvider(input.Name, options...) - Expect(clusterProvider).ToNot(BeNil(), "Failed to create a kind cluster") - - clusterProvider.Create(ctx) - Expect(clusterProvider.GetKubeconfigPath()).To(BeAnExistingFile(), "The kubeconfig file for the kind cluster with name %q does not exists at %q as expected", input.Name, clusterProvider.GetKubeconfigPath()) - - log.Logf("The kubeconfig file for the kind cluster is %s", clusterProvider.kubeconfigPath) - - err := LoadImagesToKindCluster(ctx, LoadImagesToKindClusterInput{ - Name: input.Name, - Images: input.Images, - }) - if err != nil { - clusterProvider.Dispose(ctx) - Expect(err).NotTo(HaveOccurred()) // re-surface the error to fail the test - } - - return clusterProvider -} - -// LoadImagesToKindClusterInput is the input for LoadImagesToKindCluster. -type LoadImagesToKindClusterInput struct { - // Name of the cluster - Name string - - // Images to be loaded in the cluster (this is kind specific) - Images []clusterctl.ContainerImage -} - -// LoadImagesToKindCluster provides a utility for loading images into a kind cluster. -func LoadImagesToKindCluster(ctx context.Context, input LoadImagesToKindClusterInput) error { - if ctx == nil { - return errors.New("ctx is required for LoadImagesToKindCluster") - } - if input.Name == "" { - return errors.New("Invalid argument. Name can't be empty when calling LoadImagesToKindCluster") - } - - containerRuntime, err := container.NewDockerClient() - if err != nil { - return errors.Wrap(err, "failed to get Docker runtime client") - } - ctx = container.RuntimeInto(ctx, containerRuntime) - - for _, image := range input.Images { - log.Logf("Loading image: %q", image.Name) - if err := loadImage(ctx, input.Name, image.Name); err != nil { - switch image.LoadBehavior { - case clusterctl.MustLoadImage: - return errors.Wrapf(err, "Failed to load image %q into the kind cluster %q", image.Name, input.Name) - case clusterctl.TryLoadImage: - log.Logf("[WARNING] Unable to load image %q into the kind cluster %q: %v", image.Name, input.Name, err) - } - } - } - return nil -} - -// LoadImage will put a local image onto the kind node. -// If the image doesn't exist locally we will attempt to pull it remotely. -func loadImage(ctx context.Context, cluster, image string) error { - // Save the image into a tar - dir, err := os.MkdirTemp("", "image-tar") - if err != nil { - return errors.Wrap(err, "failed to create tempdir") - } - defer os.RemoveAll(dir) - imageTarPath := filepath.Join(dir, "image.tar") - - containerRuntime, err := container.RuntimeFrom(ctx) - if err != nil { - return errors.Wrap(err, "failed to access container runtime") - } - // in the nominal E2E scenario images have been locally built and added to cache - exists, err := containerRuntime.ImageExistsLocally(ctx, image) - if err != nil { - return errors.Wrapf(err, "error listing local image %s", image) - } - // in some scenarios we refer to a real reference image which may not have been pre-downloaded - if !exists { - log.Logf("Image %s not present in local container image cache, will pull", image) - err := containerRuntime.PullContainerImage(ctx, image) - if err != nil { - return errors.Wrapf(err, "error pulling image %q", image) - } - } else { - log.Logf("Image %s is present in local container image cache", image) - } - err = containerRuntime.SaveContainerImage(ctx, image, imageTarPath) - if err != nil { - return errors.Wrapf(err, "error saving image %q to %q", image, imageTarPath) - } - - // Gets the nodes in the cluster - provider := kind.NewProvider() - nodeList, err := provider.ListInternalNodes(cluster) - if err != nil { - return err - } - - // Load the image on the selected nodes - for _, node := range nodeList { - if err := load(imageTarPath, node); err != nil { - return err - } - } - - return nil -} - -// copied from kind https://github.com/kubernetes-sigs/kind/blob/v0.7.0/pkg/cmd/kind/load/docker-image/docker-image.go#L158 -// loads an image tarball onto a node. -func load(imageTarName string, node kindnodes.Node) error { - f, err := os.Open(filepath.Clean(imageTarName)) - if err != nil { - return errors.Wrap(err, "failed to open image") - } - defer f.Close() - return kindnodesutils.LoadImageArchive(node, f) -} diff --git a/test/e2e/framework/cluster_helpers.go b/test/e2e/framework/cluster_helpers.go deleted file mode 100644 index 54c9a3bb..00000000 --- a/test/e2e/framework/cluster_helpers.go +++ /dev/null @@ -1,287 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "context" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/klog/v2" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" - "sigs.k8s.io/cluster-api/util/patch" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/kubesphere/kubekey/v3/test/e2e/framework/internal/log" -) - -// CreateClusterInput is the input for CreateCluster. -type CreateClusterInput struct { - Creator Creator - Cluster *clusterv1.Cluster - InfraCluster client.Object -} - -// CreateCluster will create the Cluster and InfraCluster objects. -func CreateCluster(ctx context.Context, input CreateClusterInput, intervals ...interface{}) { - By("creating an InfrastructureCluster resource") - Eventually(func() error { - return input.Creator.Create(ctx, input.InfraCluster) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to create InfrastructureCluster %s", klog.KObj(input.InfraCluster)) - - // This call happens in an eventually because of a race condition with the - // webhook server. If the latter isn't fully online then this call will - // fail. - By("creating a Cluster resource linked to the InfrastructureCluster resource") - Eventually(func() error { - if err := input.Creator.Create(ctx, input.Cluster); err != nil { - log.Logf("Failed to create a cluster: %+v", err) - return err - } - return nil - }, intervals...).Should(Succeed(), "Failed to create Cluster %s", klog.KObj(input.Cluster)) -} - -// GetAllClustersByNamespaceInput is the input for GetAllClustersByNamespace. -type GetAllClustersByNamespaceInput struct { - Lister Lister - Namespace string -} - -// GetAllClustersByNamespace returns the list of Cluster object in a namespace. -func GetAllClustersByNamespace(ctx context.Context, input GetAllClustersByNamespaceInput) []*clusterv1.Cluster { - clusterList := &clusterv1.ClusterList{} - Eventually(func() error { - return input.Lister.List(ctx, clusterList, client.InNamespace(input.Namespace)) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list clusters in namespace %s", input.Namespace) - - clusters := make([]*clusterv1.Cluster, len(clusterList.Items)) - for i := range clusterList.Items { - clusters[i] = &clusterList.Items[i] - } - return clusters -} - -// GetClusterByNameInput is the input for GetClusterByName. -type GetClusterByNameInput struct { - Getter Getter - Name string - Namespace string -} - -// GetClusterByName returns a Cluster object given his name. -func GetClusterByName(ctx context.Context, input GetClusterByNameInput) *clusterv1.Cluster { - cluster := &clusterv1.Cluster{} - key := client.ObjectKey{ - Namespace: input.Namespace, - Name: input.Name, - } - Eventually(func() error { - return input.Getter.Get(ctx, key, cluster) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get Cluster object %s", klog.KRef(input.Namespace, input.Name)) - return cluster -} - -// PatchClusterLabelInput is the input for PatchClusterLabel. -type PatchClusterLabelInput struct { - ClusterProxy ClusterProxy - Cluster *clusterv1.Cluster - Labels map[string]string -} - -// PatchClusterLabel patches labels to a cluster. -func PatchClusterLabel(ctx context.Context, input PatchClusterLabelInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for PatchClusterLabel") - Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling PatchClusterLabel") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling PatchClusterLabel") - Expect(input.Labels).ToNot(BeEmpty(), "Invalid argument. input.Labels can't be empty when calling PatchClusterLabel") - - log.Logf("Patching the label to the cluster") - patchHelper, err := patch.NewHelper(input.Cluster, input.ClusterProxy.GetClient()) - Expect(err).ToNot(HaveOccurred()) - input.Cluster.SetLabels(input.Labels) - Eventually(func() error { - return patchHelper.Patch(ctx, input.Cluster) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to patch label to cluster %s", klog.KObj(input.Cluster)) -} - -// WaitForClusterToProvisionInput is the input for WaitForClusterToProvision. -type WaitForClusterToProvisionInput struct { - Getter Getter - Cluster *clusterv1.Cluster -} - -// WaitForClusterToProvision will wait for a cluster to have a phase status of provisioned. -func WaitForClusterToProvision(ctx context.Context, input WaitForClusterToProvisionInput, intervals ...interface{}) *clusterv1.Cluster { - cluster := &clusterv1.Cluster{} - By("Waiting for cluster to enter the provisioned phase") - Eventually(func() (string, error) { - key := client.ObjectKey{ - Namespace: input.Cluster.GetNamespace(), - Name: input.Cluster.GetName(), - } - if err := input.Getter.Get(ctx, key, cluster); err != nil { - return "", err - } - return cluster.Status.Phase, nil - }, intervals...).Should(Equal(string(clusterv1.ClusterPhaseProvisioned)), "Timed out waiting for Cluster %s to provision", klog.KObj(input.Cluster)) - return cluster -} - -// DeleteClusterInput is the input for DeleteCluster. -type DeleteClusterInput struct { - Deleter Deleter - Cluster *clusterv1.Cluster -} - -// DeleteCluster deletes the cluster. -func DeleteCluster(ctx context.Context, input DeleteClusterInput) { - Byf("Deleting cluster %s", input.Cluster.GetName()) - Expect(input.Deleter.Delete(ctx, input.Cluster)).To(Succeed()) -} - -// WaitForClusterDeletedInput is the input for WaitForClusterDeleted. -type WaitForClusterDeletedInput struct { - Getter Getter - Cluster *clusterv1.Cluster -} - -// WaitForClusterDeleted waits until the cluster object has been deleted. -func WaitForClusterDeleted(ctx context.Context, input WaitForClusterDeletedInput, intervals ...interface{}) { - Byf("Waiting for cluster %s to be deleted", input.Cluster.GetName()) - Eventually(func() bool { - cluster := &clusterv1.Cluster{} - key := client.ObjectKey{ - Namespace: input.Cluster.GetNamespace(), - Name: input.Cluster.GetName(), - } - return apierrors.IsNotFound(input.Getter.Get(ctx, key, cluster)) - }, intervals...).Should(BeTrue()) -} - -// DiscoveryAndWaitForClusterInput is the input type for DiscoveryAndWaitForCluster. -type DiscoveryAndWaitForClusterInput struct { - Getter Getter - Namespace string - Name string -} - -// DiscoveryAndWaitForCluster discovers a cluster object in a namespace and waits for the cluster infrastructure to be provisioned. -func DiscoveryAndWaitForCluster(ctx context.Context, input DiscoveryAndWaitForClusterInput, intervals ...interface{}) *clusterv1.Cluster { - Expect(ctx).NotTo(BeNil(), "ctx is required for DiscoveryAndWaitForCluster") - Expect(input.Getter).ToNot(BeNil(), "Invalid argument. input.Getter can't be nil when calling DiscoveryAndWaitForCluster") - Expect(input.Namespace).ToNot(BeNil(), "Invalid argument. input.Namespace can't be empty when calling DiscoveryAndWaitForCluster") - Expect(input.Name).ToNot(BeNil(), "Invalid argument. input.Name can't be empty when calling DiscoveryAndWaitForCluster") - - var cluster *clusterv1.Cluster - Eventually(func() bool { - cluster = GetClusterByName(ctx, GetClusterByNameInput{ - Getter: input.Getter, - Name: input.Name, - Namespace: input.Namespace, - }) - return cluster != nil - }, retryableOperationTimeout, retryableOperationInterval).Should(BeTrue(), "Failed to get Cluster object %s", klog.KRef(input.Namespace, input.Name)) - - // NOTE: We intentionally return the provisioned Cluster because it also contains - // the reconciled ControlPlane ref and InfrastructureCluster ref when using a ClusterClass. - cluster = WaitForClusterToProvision(ctx, WaitForClusterToProvisionInput{ - Getter: input.Getter, - Cluster: cluster, - }, intervals...) - - return cluster -} - -// DeleteClusterAndWaitInput is the input type for DeleteClusterAndWait. -type DeleteClusterAndWaitInput struct { - Client client.Client - Cluster *clusterv1.Cluster -} - -// DeleteClusterAndWait deletes a cluster object and waits for it to be gone. -func DeleteClusterAndWait(ctx context.Context, input DeleteClusterAndWaitInput, intervals ...interface{}) { - Expect(ctx).NotTo(BeNil(), "ctx is required for DeleteClusterAndWait") - Expect(input.Client).ToNot(BeNil(), "Invalid argument. input.Client can't be nil when calling DeleteClusterAndWait") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling DeleteClusterAndWait") - - DeleteCluster(ctx, DeleteClusterInput{ - Deleter: input.Client, - Cluster: input.Cluster, - }) - - log.Logf("Waiting for the Cluster object to be deleted") - WaitForClusterDeleted(ctx, WaitForClusterDeletedInput{ - Getter: input.Client, - Cluster: input.Cluster, - }, intervals...) - - //TODO: consider if to move in another func (what if there are more than one cluster?) - log.Logf("Check for all the Cluster API resources being deleted") - Eventually(func() []*unstructured.Unstructured { - return GetCAPIResources(ctx, GetCAPIResourcesInput{ - Lister: input.Client, - Namespace: input.Cluster.Namespace, - }) - }, retryableOperationTimeout, retryableOperationInterval).Should(BeEmpty(), "There are still Cluster API resources in the %q namespace", input.Cluster.Namespace) -} - -// DeleteAllClustersAndWaitInput is the input type for DeleteAllClustersAndWait. -type DeleteAllClustersAndWaitInput struct { - Client client.Client - Namespace string -} - -// DeleteAllClustersAndWait deletes a cluster object and waits for it to be gone. -func DeleteAllClustersAndWait(ctx context.Context, input DeleteAllClustersAndWaitInput, intervals ...interface{}) { - Expect(ctx).NotTo(BeNil(), "ctx is required for DeleteAllClustersAndWait") - Expect(input.Client).ToNot(BeNil(), "Invalid argument. input.Client can't be nil when calling DeleteAllClustersAndWait") - Expect(input.Namespace).ToNot(BeEmpty(), "Invalid argument. input.Namespace can't be empty when calling DeleteAllClustersAndWait") - - clusters := GetAllClustersByNamespace(ctx, GetAllClustersByNamespaceInput{ - Lister: input.Client, - Namespace: input.Namespace, - }) - - for _, c := range clusters { - DeleteCluster(ctx, DeleteClusterInput{ - Deleter: input.Client, - Cluster: c, - }) - } - - for _, c := range clusters { - log.Logf("Waiting for the Cluster %s to be deleted", klog.KObj(c)) - WaitForClusterDeleted(ctx, WaitForClusterDeletedInput{ - Getter: input.Client, - Cluster: c, - }, intervals...) - } -} - -// byClusterOptions returns a set of ListOptions that allows to identify all the objects belonging to a Cluster. -func byClusterOptions(name, namespace string) []client.ListOption { - return []client.ListOption{ - client.InNamespace(namespace), - client.MatchingLabels{ - clusterv1.ClusterLabelName: name, - }, - } -} diff --git a/test/e2e/framework/cluster_proxy.go b/test/e2e/framework/cluster_proxy.go deleted file mode 100644 index 9759c6da..00000000 --- a/test/e2e/framework/cluster_proxy.go +++ /dev/null @@ -1,377 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "context" - "errors" - "fmt" - "net/url" - "os" - "path" - goruntime "runtime" - "time" - - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/clientcmd/api" - "k8s.io/klog/v2" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" - "sigs.k8s.io/cluster-api/test/framework/exec" - "sigs.k8s.io/cluster-api/test/infrastructure/container" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/kubesphere/kubekey/v3/test/e2e/framework/internal/log" -) - -const ( - retryableOperationInterval = 3 * time.Second - // retryableOperationTimeout requires a higher value especially for self-hosted upgrades. - // Short unavailability of the Kube APIServer due to joining etcd members paired with unreachable conversion webhooks due to - // failed leader election and thus controller restarts lead to longer taking retries. - // The timeout occurs when listing machines in `GetControlPlaneMachinesByCluster`. - retryableOperationTimeout = 3 * time.Minute -) - -// ClusterProxy defines the behavior of a type that acts as an intermediary with an existing Kubernetes cluster. -// It should work with any Kubernetes cluster, no matter if the Cluster was created by a bootstrap.ClusterProvider, -// by Cluster API (a workload cluster or a self-hosted cluster) or else. -type ClusterProxy interface { - // GetName returns the name of the cluster. - GetName() string - - // GetKubeconfigPath returns the path to the kubeconfig file to be used to access the Kubernetes cluster. - GetKubeconfigPath() string - - // GetScheme returns the scheme defining the types hosted in the Kubernetes cluster. - // It is used when creating a controller-runtime client. - GetScheme() *runtime.Scheme - - // GetClient returns a controller-runtime client to the Kubernetes cluster. - GetClient() client.Client - - // GetClientSet returns a client-go client to the Kubernetes cluster. - GetClientSet() *kubernetes.Clientset - - // GetRESTConfig returns the REST config for direct use with client-go if needed. - GetRESTConfig() *rest.Config - - // GetLogCollector returns the machine log collector for the Kubernetes cluster. - GetLogCollector() ClusterLogCollector - - // Apply to apply YAML to the Kubernetes cluster, `kubectl apply`. - Apply(ctx context.Context, resources []byte, args ...string) error - - // GetWorkloadCluster returns a proxy to a workload cluster defined in the Kubernetes cluster. - GetWorkloadCluster(ctx context.Context, namespace, name string) ClusterProxy - - // CollectWorkloadClusterLogs collects machines logs from the workload cluster. - CollectWorkloadClusterLogs(ctx context.Context, namespace, name, outputPath string) - - // Dispose proxy's internal resources (the operation does not affects the Kubernetes cluster). - // This should be implemented as a synchronous function. - Dispose(context.Context) -} - -// ClusterLogCollector defines an object that can collect logs from a machine. -type ClusterLogCollector interface { - // CollectMachineLog collects log from a machine. - // TODO: describe output folder struct - CollectMachineLog(ctx context.Context, managementClusterClient client.Client, m *clusterv1.Machine, outputPath string) error - CollectMachinePoolLog(ctx context.Context, managementClusterClient client.Client, m *expv1.MachinePool, outputPath string) error -} - -// Option is a configuration option supplied to NewClusterProxy. -type Option func(*clusterProxy) - -// WithMachineLogCollector allows to define the machine log collector to be used with this Cluster. -func WithMachineLogCollector(logCollector ClusterLogCollector) Option { - return func(c *clusterProxy) { - c.logCollector = logCollector - } -} - -// clusterProxy provides a base implementation of the ClusterProxy interface. -type clusterProxy struct { - name string - kubeconfigPath string - scheme *runtime.Scheme - shouldCleanupKubeconfig bool - logCollector ClusterLogCollector -} - -// NewClusterProxy returns a clusterProxy given a KubeconfigPath and the scheme defining the types hosted in the cluster. -// If a kubeconfig file isn't provided, standard kubeconfig locations will be used (kubectl loading rules apply). -func NewClusterProxy(name string, kubeconfigPath string, scheme *runtime.Scheme, options ...Option) ClusterProxy { - Expect(scheme).NotTo(BeNil(), "scheme is required for NewClusterProxy") - - if kubeconfigPath == "" { - kubeconfigPath = clientcmd.NewDefaultClientConfigLoadingRules().GetDefaultFilename() - } - - proxy := &clusterProxy{ - name: name, - kubeconfigPath: kubeconfigPath, - scheme: scheme, - shouldCleanupKubeconfig: false, - } - - for _, o := range options { - o(proxy) - } - - return proxy -} - -// newFromAPIConfig returns a clusterProxy given a api.Config and the scheme defining the types hosted in the cluster. -func newFromAPIConfig(name string, config *api.Config, scheme *runtime.Scheme) ClusterProxy { - // NB. the ClusterProvider is responsible for the cleanup of this file - f, err := os.CreateTemp("", "e2e-kubeconfig") - Expect(err).ToNot(HaveOccurred(), "Failed to create kubeconfig file for the kind cluster %q") - kubeconfigPath := f.Name() - - err = clientcmd.WriteToFile(*config, kubeconfigPath) - Expect(err).ToNot(HaveOccurred(), "Failed to write kubeconfig for the kind cluster to a file %q") - - return &clusterProxy{ - name: name, - kubeconfigPath: kubeconfigPath, - scheme: scheme, - shouldCleanupKubeconfig: true, - } -} - -// GetName returns the name of the cluster. -func (p *clusterProxy) GetName() string { - return p.name -} - -// GetKubeconfigPath returns the path to the kubeconfig file for the cluster. -func (p *clusterProxy) GetKubeconfigPath() string { - return p.kubeconfigPath -} - -// GetScheme returns the scheme defining the types hosted in the cluster. -func (p *clusterProxy) GetScheme() *runtime.Scheme { - return p.scheme -} - -// GetClient returns a controller-runtime client for the cluster. -func (p *clusterProxy) GetClient() client.Client { - config := p.GetRESTConfig() - - var c client.Client - var newClientErr error - err := wait.PollImmediate(retryableOperationInterval, retryableOperationTimeout, func() (bool, error) { - c, newClientErr = client.New(config, client.Options{Scheme: p.scheme}) - if newClientErr != nil { - return false, nil //nolint:nilerr - } - return true, nil - }) - errorString := "Failed to get controller-runtime client" - Expect(newClientErr).ToNot(HaveOccurred(), errorString) - Expect(err).ToNot(HaveOccurred(), errorString) - - return c -} - -// GetClientSet returns a client-go client for the cluster. -func (p *clusterProxy) GetClientSet() *kubernetes.Clientset { - restConfig := p.GetRESTConfig() - - cs, err := kubernetes.NewForConfig(restConfig) - Expect(err).ToNot(HaveOccurred(), "Failed to get client-go client") - - return cs -} - -// Apply wraps `kubectl apply ...` and prints the output so we can see what gets applied to the cluster. -func (p *clusterProxy) Apply(ctx context.Context, resources []byte, args ...string) error { - Expect(ctx).NotTo(BeNil(), "ctx is required for Apply") - Expect(resources).NotTo(BeNil(), "resources is required for Apply") - - return exec.KubectlApply(ctx, p.kubeconfigPath, resources, args...) -} - -func (p *clusterProxy) GetRESTConfig() *rest.Config { - config, err := clientcmd.LoadFromFile(p.kubeconfigPath) - Expect(err).ToNot(HaveOccurred(), "Failed to load Kubeconfig file from %q", p.kubeconfigPath) - - restConfig, err := clientcmd.NewDefaultClientConfig(*config, &clientcmd.ConfigOverrides{}).ClientConfig() - Expect(err).ToNot(HaveOccurred(), "Failed to get ClientConfig from %q", p.kubeconfigPath) - - restConfig.UserAgent = "cluster-api-e2e" - return restConfig -} - -func (p *clusterProxy) GetLogCollector() ClusterLogCollector { - return p.logCollector -} - -// GetWorkloadCluster returns ClusterProxy for the workload cluster. -func (p *clusterProxy) GetWorkloadCluster(ctx context.Context, namespace, name string) ClusterProxy { - Expect(ctx).NotTo(BeNil(), "ctx is required for GetWorkloadCluster") - Expect(namespace).NotTo(BeEmpty(), "namespace is required for GetWorkloadCluster") - Expect(name).NotTo(BeEmpty(), "name is required for GetWorkloadCluster") - - // gets the kubeconfig from the cluster - config := p.getKubeconfig(ctx, namespace, name) - - // if we are on mac and the cluster is a DockerCluster, it is required to fix the control plane address - // by using localhost:load-balancer-host-port instead of the address used in the docker network. - if goruntime.GOOS == "darwin" && p.isDockerCluster(ctx, namespace, name) { - p.fixConfig(ctx, name, config) - } - - return newFromAPIConfig(name, config, p.scheme) -} - -// CollectWorkloadClusterLogs collects machines logs from the workload cluster. -func (p *clusterProxy) CollectWorkloadClusterLogs(ctx context.Context, namespace, name, outputPath string) { - if p.logCollector == nil { - return - } - - var machines *clusterv1.MachineList - Eventually(func() error { - var err error - machines, err = getMachinesInCluster(ctx, p.GetClient(), namespace, name) - return err - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get Machines for the Cluster %s", klog.KRef(namespace, name)) - - for i := range machines.Items { - m := &machines.Items[i] - err := p.logCollector.CollectMachineLog(ctx, p.GetClient(), m, path.Join(outputPath, "machines", m.GetName())) - if err != nil { - // NB. we are treating failures in collecting logs as a non blocking operation (best effort) - fmt.Printf("Failed to get logs for Machine %s, Cluster %s: %v\n", m.GetName(), klog.KRef(namespace, name), err) - } - } - - var machinePools *expv1.MachinePoolList - Eventually(func() error { - var err error - machinePools, err = getMachinePoolsInCluster(ctx, p.GetClient(), namespace, name) - return err - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get MachinePools for Cluster %s", klog.KRef(namespace, name)) - - for i := range machinePools.Items { - mp := &machinePools.Items[i] - err := p.logCollector.CollectMachinePoolLog(ctx, p.GetClient(), mp, path.Join(outputPath, "machine-pools", mp.GetName())) - if err != nil { - // NB. we are treating failures in collecting logs as a non blocking operation (best effort) - fmt.Printf("Failed to get logs for MachinePool %s, Cluster %s: %v\n", mp.GetName(), klog.KRef(namespace, name), err) - } - } -} - -func getMachinesInCluster(ctx context.Context, c client.Client, namespace, name string) (*clusterv1.MachineList, error) { - if name == "" { - return nil, errors.New("cluster name should not be empty") - } - - machineList := &clusterv1.MachineList{} - labels := map[string]string{clusterv1.ClusterLabelName: name} - if err := c.List(ctx, machineList, client.InNamespace(namespace), client.MatchingLabels(labels)); err != nil { - return nil, err - } - - return machineList, nil -} - -func getMachinePoolsInCluster(ctx context.Context, c client.Client, namespace, name string) (*expv1.MachinePoolList, error) { - if name == "" { - return nil, errors.New("cluster name should not be empty") - } - - machinePoolList := &expv1.MachinePoolList{} - labels := map[string]string{clusterv1.ClusterLabelName: name} - if err := c.List(ctx, machinePoolList, client.InNamespace(namespace), client.MatchingLabels(labels)); err != nil { - return nil, err - } - - return machinePoolList, nil -} - -func (p *clusterProxy) getKubeconfig(ctx context.Context, namespace string, name string) *api.Config { - cl := p.GetClient() - - secret := &corev1.Secret{} - key := client.ObjectKey{ - Name: fmt.Sprintf("%s-kubeconfig", name), - Namespace: namespace, - } - Eventually(func() error { - return cl.Get(ctx, key, secret) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get %s", key) - Expect(secret.Data).To(HaveKey("value"), "Invalid secret %s", key) - - config, err := clientcmd.Load(secret.Data["value"]) - Expect(err).ToNot(HaveOccurred(), "Failed to convert %s into a kubeconfig file", key) - - return config -} - -func (p *clusterProxy) isDockerCluster(ctx context.Context, namespace string, name string) bool { - cl := p.GetClient() - - cluster := &clusterv1.Cluster{} - key := client.ObjectKey{ - Name: name, - Namespace: namespace, - } - Eventually(func() error { - return cl.Get(ctx, key, cluster) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get %s", key) - - return cluster.Spec.InfrastructureRef.Kind == "DockerCluster" -} - -func (p *clusterProxy) fixConfig(ctx context.Context, name string, config *api.Config) { - containerRuntime, err := container.NewDockerClient() - Expect(err).ToNot(HaveOccurred(), "Failed to get Docker runtime client") - ctx = container.RuntimeInto(ctx, containerRuntime) - - lbContainerName := name + "-lb" - port, err := containerRuntime.GetHostPort(ctx, lbContainerName, "6443/tcp") - Expect(err).ToNot(HaveOccurred(), "Failed to get load balancer port") - - controlPlaneURL := &url.URL{ - Scheme: "https", - Host: "127.0.0.1:" + port, - } - currentCluster := config.Contexts[config.CurrentContext].Cluster - config.Clusters[currentCluster].Server = controlPlaneURL.String() -} - -// Dispose clusterProxy internal resources (the operation does not affects the Kubernetes cluster). -func (p *clusterProxy) Dispose(ctx context.Context) { - Expect(ctx).NotTo(BeNil(), "ctx is required for Dispose") - - if p.shouldCleanupKubeconfig { - if err := os.Remove(p.kubeconfigPath); err != nil { - log.Logf("Deleting the kubeconfig file %q file. You may need to remove this by hand.", p.kubeconfigPath) - } - } -} diff --git a/test/e2e/framework/cluster_topology_helpers.go b/test/e2e/framework/cluster_topology_helpers.go deleted file mode 100644 index 95691258..00000000 --- a/test/e2e/framework/cluster_topology_helpers.go +++ /dev/null @@ -1,174 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "context" - "strconv" - - . "github.com/onsi/gomega" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/klog/v2" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/util/patch" - "sigs.k8s.io/controller-runtime/pkg/client" - - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" - "github.com/kubesphere/kubekey/v3/test/e2e/framework/internal/log" -) - -// GetClusterClassByNameInput is the input for GetClusterClassByName. -type GetClusterClassByNameInput struct { - Getter Getter - Name string - Namespace string -} - -// GetClusterClassByName returns a ClusterClass object given his name and namespace. -func GetClusterClassByName(ctx context.Context, input GetClusterClassByNameInput) *clusterv1.ClusterClass { - Expect(ctx).NotTo(BeNil(), "ctx is required for GetClusterClassByName") - Expect(input.Getter).ToNot(BeNil(), "Invalid argument. input.Getter can't be nil when calling GetClusterClassByName") - Expect(input.Namespace).ToNot(BeNil(), "Invalid argument. input.Namespace can't be empty when calling GetClusterClassByName") - Expect(input.Name).ToNot(BeNil(), "Invalid argument. input.Name can't be empty when calling GetClusterClassByName") - - clusterClass := &clusterv1.ClusterClass{} - key := client.ObjectKey{ - Namespace: input.Namespace, - Name: input.Name, - } - Eventually(func() error { - return input.Getter.Get(ctx, key, clusterClass) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get ClusterClass object %s", klog.KRef(input.Namespace, input.Name)) - return clusterClass -} - -// UpgradeClusterTopologyAndWaitForUpgradeInput is the input type for UpgradeClusterTopologyAndWaitForUpgrade. -type UpgradeClusterTopologyAndWaitForUpgradeInput struct { - ClusterProxy ClusterProxy - Cluster *clusterv1.Cluster - ControlPlane *infracontrolplanev1.K3sControlPlane - EtcdImageTag string - DNSImageTag string - MachineDeployments []*clusterv1.MachineDeployment - KubernetesUpgradeVersion string - WaitForMachinesToBeUpgraded []interface{} - WaitForKubeProxyUpgrade []interface{} - WaitForDNSUpgrade []interface{} - WaitForEtcdUpgrade []interface{} - PreWaitForControlPlaneToBeUpgraded func() - PreWaitForMachineDeploymentToBeUpgraded func() -} - -// UpgradeClusterTopologyAndWaitForUpgrade upgrades a Cluster topology and waits for it to be upgraded. -// NOTE: This func only works with K3sControlPlane. -func UpgradeClusterTopologyAndWaitForUpgrade(ctx context.Context, input UpgradeClusterTopologyAndWaitForUpgradeInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for UpgradeClusterTopologyAndWaitForUpgrade") - Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling UpgradeClusterTopologyAndWaitForUpgrade") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling UpgradeClusterTopologyAndWaitForUpgrade") - Expect(input.ControlPlane).ToNot(BeNil(), "Invalid argument. input.ControlPlane can't be nil when calling UpgradeClusterTopologyAndWaitForUpgrade") - Expect(input.MachineDeployments).ToNot(BeEmpty(), "Invalid argument. input.MachineDeployments can't be empty when calling UpgradeClusterTopologyAndWaitForUpgrade") - Expect(input.KubernetesUpgradeVersion).ToNot(BeNil(), "Invalid argument. input.KubernetesUpgradeVersion can't be empty when calling UpgradeClusterTopologyAndWaitForUpgrade") - - mgmtClient := input.ClusterProxy.GetClient() - - log.Logf("Patching the new Kubernetes version to Cluster topology") - patchHelper, err := patch.NewHelper(input.Cluster, mgmtClient) - Expect(err).ToNot(HaveOccurred()) - - input.Cluster.Spec.Topology.Version = input.KubernetesUpgradeVersion - for i, variable := range input.Cluster.Spec.Topology.Variables { - if variable.Name == "etcdImageTag" { - // NOTE: strconv.Quote is used to produce a valid JSON string. - input.Cluster.Spec.Topology.Variables[i].Value = apiextensionsv1.JSON{Raw: []byte(strconv.Quote(input.EtcdImageTag))} - } - if variable.Name == "coreDNSImageTag" { - // NOTE: strconv.Quote is used to produce a valid JSON string. - input.Cluster.Spec.Topology.Variables[i].Value = apiextensionsv1.JSON{Raw: []byte(strconv.Quote(input.DNSImageTag))} - } - } - Eventually(func() error { - return patchHelper.Patch(ctx, input.Cluster) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to patch Cluster topology %s with version %s", klog.KObj(input.Cluster), input.KubernetesUpgradeVersion) - - // Once we have patched the Kubernetes Cluster we can run PreWaitForControlPlaneToBeUpgraded. - // Note: This can e.g. be used to verify the BeforeClusterUpgrade lifecycle hook is executed - // and blocking correctly. - if input.PreWaitForControlPlaneToBeUpgraded != nil { - log.Logf("Calling PreWaitForControlPlaneToBeUpgraded") - input.PreWaitForControlPlaneToBeUpgraded() - } - - log.Logf("Waiting for control-plane machines to have the upgraded Kubernetes version") - WaitForControlPlaneMachinesToBeUpgraded(ctx, WaitForControlPlaneMachinesToBeUpgradedInput{ - Lister: mgmtClient, - Cluster: input.Cluster, - MachineCount: int(*input.ControlPlane.Spec.Replicas), - KubernetesUpgradeVersion: input.KubernetesUpgradeVersion, - }, input.WaitForMachinesToBeUpgraded...) - - log.Logf("Waiting for kube-proxy to have the upgraded Kubernetes version") - workloadCluster := input.ClusterProxy.GetWorkloadCluster(ctx, input.Cluster.Namespace, input.Cluster.Name) - workloadClient := workloadCluster.GetClient() - WaitForKubeProxyUpgrade(ctx, WaitForKubeProxyUpgradeInput{ - Getter: workloadClient, - KubernetesVersion: input.KubernetesUpgradeVersion, - }, input.WaitForKubeProxyUpgrade...) - - // Wait for the CoreDNS upgrade if the DNSImageTag is set. - if input.DNSImageTag != "" { - log.Logf("Waiting for CoreDNS to have the upgraded image tag") - WaitForDNSUpgrade(ctx, WaitForDNSUpgradeInput{ - Getter: workloadClient, - DNSVersion: input.DNSImageTag, - }, input.WaitForDNSUpgrade...) - } - - // Wait for the etcd upgrade if the EtcdImageTag is set. - if input.EtcdImageTag != "" { - log.Logf("Waiting for etcd to have the upgraded image tag") - lblSelector, err := labels.Parse("component=etcd") - Expect(err).ToNot(HaveOccurred()) - WaitForPodListCondition(ctx, WaitForPodListConditionInput{ - Lister: workloadClient, - ListOptions: &client.ListOptions{LabelSelector: lblSelector}, - Condition: EtcdImageTagCondition(input.EtcdImageTag, int(*input.ControlPlane.Spec.Replicas)), - }, input.WaitForEtcdUpgrade...) - } - - // Once the ControlPlane is upgraded we can run PreWaitForMachineDeploymentToBeUpgraded. - // Note: This can e.g. be used to verify the AfterControlPlaneUpgrade lifecycle hook is executed - // and blocking correctly. - if input.PreWaitForMachineDeploymentToBeUpgraded != nil { - log.Logf("Calling PreWaitForMachineDeploymentToBeUpgraded") - input.PreWaitForMachineDeploymentToBeUpgraded() - } - - for _, deployment := range input.MachineDeployments { - if *deployment.Spec.Replicas > 0 { - log.Logf("Waiting for Kubernetes versions of machines in MachineDeployment %s to be upgraded to %s", - klog.KObj(deployment), input.KubernetesUpgradeVersion) - WaitForMachineDeploymentMachinesToBeUpgraded(ctx, WaitForMachineDeploymentMachinesToBeUpgradedInput{ - Lister: mgmtClient, - Cluster: input.Cluster, - MachineCount: int(*deployment.Spec.Replicas), - KubernetesUpgradeVersion: input.KubernetesUpgradeVersion, - MachineDeployment: *deployment, - }, input.WaitForMachinesToBeUpgraded...) - } - } -} diff --git a/test/e2e/framework/clusterctl/client.go b/test/e2e/framework/clusterctl/client.go deleted file mode 100644 index ab8b8bae..00000000 --- a/test/e2e/framework/clusterctl/client.go +++ /dev/null @@ -1,418 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package clusterctl - -import ( - "context" - "fmt" - "os" - "os/exec" - "path" - "path/filepath" - "strings" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - clusterctlclient "sigs.k8s.io/cluster-api/cmd/clusterctl/client" - clusterctllog "sigs.k8s.io/cluster-api/cmd/clusterctl/log" - - "github.com/kubesphere/kubekey/v3/test/e2e/framework/clusterctl/logger" - "github.com/kubesphere/kubekey/v3/test/e2e/framework/internal/log" -) - -// Provide E2E friendly wrappers for the clusterctl client library. - -const ( - // DefaultFlavor for ConfigClusterInput; use it for getting the cluster-template.yaml file. - DefaultFlavor = "" -) - -const ( - // DefaultInfrastructureProvider for ConfigClusterInput; use it for using the only infrastructure provider installed in a cluster. - DefaultInfrastructureProvider = "" -) - -// InitInput is the input for Init. -type InitInput struct { - LogFolder string - ClusterctlConfigPath string - KubeconfigPath string - CoreProvider string - BootstrapProviders []string - ControlPlaneProviders []string - InfrastructureProviders []string -} - -// Init calls clusterctl init with the list of providers defined in the local repository. -func Init(_ context.Context, input InitInput) { - log.Logf("clusterctl init --core %s --bootstrap %s --control-plane %s --infrastructure %s --config %s --kubeconfig %s", - input.CoreProvider, - strings.Join(input.BootstrapProviders, ","), - strings.Join(input.ControlPlaneProviders, ","), - strings.Join(input.InfrastructureProviders, ","), - input.ClusterctlConfigPath, - input.KubeconfigPath, - ) - - initOpt := clusterctlclient.InitOptions{ - Kubeconfig: clusterctlclient.Kubeconfig{ - Path: input.KubeconfigPath, - Context: "", - }, - CoreProvider: input.CoreProvider, - BootstrapProviders: input.BootstrapProviders, - ControlPlaneProviders: input.ControlPlaneProviders, - InfrastructureProviders: input.InfrastructureProviders, - LogUsageInstructions: true, - WaitProviders: true, - } - - clusterctlClient, log := getClusterctlClientWithLogger(input.ClusterctlConfigPath, "clusterctl-init.log", input.LogFolder) - defer log.Close() - - _, err := clusterctlClient.Init(initOpt) - Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl init") -} - -// InitWithBinary uses clusterctl binary to run init with the list of providers defined in the local repository. -func InitWithBinary(_ context.Context, binary string, input InitInput) { - log.Logf("clusterctl init --core %s --bootstrap %s --control-plane %s --infrastructure %s --config %s --kubeconfig %s", - input.CoreProvider, - strings.Join(input.BootstrapProviders, ","), - strings.Join(input.ControlPlaneProviders, ","), - strings.Join(input.InfrastructureProviders, ","), - input.ClusterctlConfigPath, - input.KubeconfigPath, - ) - - args := []string{"init", "--config", input.ClusterctlConfigPath, "--kubeconfig", input.KubeconfigPath} - if input.CoreProvider != "" { - args = append(args, "--core", input.CoreProvider) - } - if len(input.BootstrapProviders) > 0 { - args = append(args, "--bootstrap", strings.Join(input.BootstrapProviders, ",")) - } - if len(input.InfrastructureProviders) > 0 { - args = append(args, "--infrastructure", strings.Join(input.InfrastructureProviders, ",")) - } - - cmd := exec.Command(binary, args...) //nolint:gosec // We don't care about command injection here. - - out, err := cmd.CombinedOutput() - _ = os.WriteFile(filepath.Join(input.LogFolder, "clusterctl-init.log"), out, 0644) //nolint:gosec // this is a log file to be shared via prow artifacts - var stdErr string - if err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - stdErr = string(exitErr.Stderr) - } - } - Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl init:\nstdout:\n%s\nstderr:\n%s", string(out), stdErr) -} - -// UpgradeInput is the input for Upgrade. -type UpgradeInput struct { - LogFolder string - ClusterctlConfigPath string - ClusterctlVariables map[string]string - ClusterName string - KubeconfigPath string - Contract string - CoreProvider string - BootstrapProviders []string - ControlPlaneProviders []string - InfrastructureProviders []string - IPAMProviders []string - RuntimeExtensionProviders []string -} - -// Upgrade calls clusterctl upgrade apply with the list of providers defined in the local repository. -func Upgrade(ctx context.Context, input UpgradeInput) { - if len(input.ClusterctlVariables) > 0 { - outputPath := filepath.Join(filepath.Dir(input.ClusterctlConfigPath), fmt.Sprintf("clusterctl-upgrade-config-%s.yaml", input.ClusterName)) - copyAndAmendClusterctlConfig(ctx, copyAndAmendClusterctlConfigInput{ - ClusterctlConfigPath: input.ClusterctlConfigPath, - OutputPath: outputPath, - Variables: input.ClusterctlVariables, - }) - input.ClusterctlConfigPath = outputPath - } - - // Check if the user want a custom upgrade - isCustomUpgrade := input.CoreProvider != "" || - len(input.BootstrapProviders) > 0 || - len(input.ControlPlaneProviders) > 0 || - len(input.InfrastructureProviders) > 0 || - len(input.IPAMProviders) > 0 || - len(input.RuntimeExtensionProviders) > 0 - - Expect((input.Contract != "" && !isCustomUpgrade) || (input.Contract == "" && isCustomUpgrade)).To(BeTrue(), `Invalid arguments. Either the input.Contract parameter or at least one of the following providers has to be set: - input.CoreProvider, input.BootstrapProviders, input.ControlPlaneProviders, input.InfrastructureProviders, input.IPAMProviders, input.RuntimeExtensionProviders`) - - if isCustomUpgrade { - log.Logf("clusterctl upgrade apply --core %s --bootstrap %s --control-plane %s --infrastructure %s --ipam %s --runtime-extension %s --config %s --kubeconfig %s", - input.CoreProvider, - strings.Join(input.BootstrapProviders, ","), - strings.Join(input.ControlPlaneProviders, ","), - strings.Join(input.InfrastructureProviders, ","), - strings.Join(input.IPAMProviders, ","), - strings.Join(input.RuntimeExtensionProviders, ","), - input.ClusterctlConfigPath, - input.KubeconfigPath, - ) - } else { - log.Logf("clusterctl upgrade apply --contract %s --config %s --kubeconfig %s", - input.Contract, - input.ClusterctlConfigPath, - input.KubeconfigPath, - ) - } - - upgradeOpt := clusterctlclient.ApplyUpgradeOptions{ - Kubeconfig: clusterctlclient.Kubeconfig{ - Path: input.KubeconfigPath, - Context: "", - }, - Contract: input.Contract, - CoreProvider: input.CoreProvider, - BootstrapProviders: input.BootstrapProviders, - ControlPlaneProviders: input.ControlPlaneProviders, - InfrastructureProviders: input.InfrastructureProviders, - WaitProviders: true, - } - - clusterctlClient, log := getClusterctlClientWithLogger(input.ClusterctlConfigPath, "clusterctl-upgrade.log", input.LogFolder) - defer log.Close() - - err := clusterctlClient.ApplyUpgrade(upgradeOpt) - Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl upgrade") -} - -// DeleteInput is the input for Delete. -type DeleteInput struct { - LogFolder string - ClusterctlConfigPath string - KubeconfigPath string -} - -// Delete calls clusterctl delete --all. -func Delete(_ context.Context, input DeleteInput) { - log.Logf("clusterctl delete --all") - - deleteOpts := clusterctlclient.DeleteOptions{ - Kubeconfig: clusterctlclient.Kubeconfig{ - Path: input.KubeconfigPath, - Context: "", - }, - DeleteAll: true, - } - - clusterctlClient, log := getClusterctlClientWithLogger(input.ClusterctlConfigPath, "clusterctl-delete.log", input.LogFolder) - defer log.Close() - - err := clusterctlClient.Delete(deleteOpts) - Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl upgrade") -} - -// ConfigClusterInput is the input for ConfigCluster. -type ConfigClusterInput struct { - LogFolder string - ClusterctlConfigPath string - KubeconfigPath string - InfrastructureProvider string - Namespace string - ClusterName string - KubernetesVersion string - ControlPlaneMachineCount *int64 - WorkerMachineCount *int64 - Flavor string - ClusterctlVariables map[string]string -} - -// ConfigCluster gets a workload cluster based on a template. -func ConfigCluster(ctx context.Context, input ConfigClusterInput) []byte { - log.Logf("clusterctl config cluster %s --infrastructure %s --kubernetes-version %s --control-plane-machine-count %d --worker-machine-count %d --flavor %s", - input.ClusterName, - valueOrDefault(input.InfrastructureProvider), - input.KubernetesVersion, - *input.ControlPlaneMachineCount, - *input.WorkerMachineCount, - valueOrDefault(input.Flavor), - ) - - templateOptions := clusterctlclient.GetClusterTemplateOptions{ - Kubeconfig: clusterctlclient.Kubeconfig{ - Path: input.KubeconfigPath, - Context: "", - }, - ProviderRepositorySource: &clusterctlclient.ProviderRepositorySourceOptions{ - InfrastructureProvider: input.InfrastructureProvider, - Flavor: input.Flavor, - }, - ClusterName: input.ClusterName, - KubernetesVersion: input.KubernetesVersion, - ControlPlaneMachineCount: input.ControlPlaneMachineCount, - WorkerMachineCount: input.WorkerMachineCount, - TargetNamespace: input.Namespace, - } - - if len(input.ClusterctlVariables) > 0 { - outputPath := filepath.Join(filepath.Dir(input.ClusterctlConfigPath), fmt.Sprintf("clusterctl-upgrade-config-%s.yaml", input.ClusterName)) - copyAndAmendClusterctlConfig(ctx, copyAndAmendClusterctlConfigInput{ - ClusterctlConfigPath: input.ClusterctlConfigPath, - OutputPath: outputPath, - Variables: input.ClusterctlVariables, - }) - input.ClusterctlConfigPath = outputPath - } - - clusterctlClient, log := getClusterctlClientWithLogger(input.ClusterctlConfigPath, fmt.Sprintf("%s-cluster-template.yaml", input.ClusterName), input.LogFolder) - defer log.Close() - - template, err := clusterctlClient.GetClusterTemplate(templateOptions) - Expect(err).ToNot(HaveOccurred(), "Failed to run clusterctl config cluster") - - yaml, err := template.Yaml() - Expect(err).ToNot(HaveOccurred(), "Failed to generate yaml for the workload cluster template") - - _, _ = log.WriteString(string(yaml)) - return yaml -} - -// ConfigClusterWithBinary uses clusterctl binary to run config cluster or generate cluster. -// NOTE: This func detects the clusterctl version and uses config cluster or generate cluster -// accordingly. We can drop the detection when we don't have to support clusterctl v0.3.x anymore. -func ConfigClusterWithBinary(_ context.Context, clusterctlBinaryPath string, input ConfigClusterInput) []byte { - log.Logf("Detect clusterctl version via: clusterctl version") - - out, err := exec.Command(clusterctlBinaryPath, "version").Output() - Expect(err).ToNot(HaveOccurred(), "error running clusterctl version") - var clusterctlSupportsGenerateCluster bool - if strings.Contains(string(out), "Major:\"1\"") { - log.Logf("Detected clusterctl v1.x") - clusterctlSupportsGenerateCluster = true - } - - var cmd *exec.Cmd - if clusterctlSupportsGenerateCluster { - log.Logf("clusterctl generate cluster %s --infrastructure %s --kubernetes-version %s --control-plane-machine-count %d --worker-machine-count %d --flavor %s", - input.ClusterName, - valueOrDefault(input.InfrastructureProvider), - input.KubernetesVersion, - *input.ControlPlaneMachineCount, - *input.WorkerMachineCount, - valueOrDefault(input.Flavor), - ) - cmd = exec.Command(clusterctlBinaryPath, "generate", "cluster", //nolint:gosec // We don't care about command injection here. - input.ClusterName, - "--infrastructure", input.InfrastructureProvider, - "--kubernetes-version", input.KubernetesVersion, - "--control-plane-machine-count", fmt.Sprint(*input.ControlPlaneMachineCount), - "--worker-machine-count", fmt.Sprint(*input.WorkerMachineCount), - "--flavor", input.Flavor, - "--target-namespace", input.Namespace, - "--config", input.ClusterctlConfigPath, - "--kubeconfig", input.KubeconfigPath, - ) - } else { - log.Logf("clusterctl config cluster %s --infrastructure %s --kubernetes-version %s --control-plane-machine-count %d --worker-machine-count %d --flavor %s", - input.ClusterName, - valueOrDefault(input.InfrastructureProvider), - input.KubernetesVersion, - *input.ControlPlaneMachineCount, - *input.WorkerMachineCount, - valueOrDefault(input.Flavor), - ) - cmd = exec.Command(clusterctlBinaryPath, "config", "cluster", //nolint:gosec // We don't care about command injection here. - input.ClusterName, - "--infrastructure", input.InfrastructureProvider, - "--kubernetes-version", input.KubernetesVersion, - "--control-plane-machine-count", fmt.Sprint(*input.ControlPlaneMachineCount), - "--worker-machine-count", fmt.Sprint(*input.WorkerMachineCount), - "--flavor", input.Flavor, - "--target-namespace", input.Namespace, - "--config", input.ClusterctlConfigPath, - "--kubeconfig", input.KubeconfigPath, - ) - } - - out, err = cmd.Output() - _ = os.WriteFile(filepath.Join(input.LogFolder, fmt.Sprintf("%s-cluster-template.yaml", input.ClusterName)), out, 0644) //nolint:gosec // this is a log file to be shared via prow artifacts - var stdErr string - if err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - stdErr = string(exitErr.Stderr) - } - } - Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl config cluster:\nstdout:\n%s\nstderr:\n%s", string(out), stdErr) - - return out -} - -// MoveInput is the input for ClusterctlMove. -type MoveInput struct { - LogFolder string - ClusterctlConfigPath string - FromKubeconfigPath string - ToKubeconfigPath string - Namespace string -} - -// Move moves workload clusters. -func Move(ctx context.Context, input MoveInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for Move") - Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling Move") - Expect(input.FromKubeconfigPath).To(BeAnExistingFile(), "Invalid argument. input.FromKubeconfigPath must be an existing file when calling Move") - Expect(input.ToKubeconfigPath).To(BeAnExistingFile(), "Invalid argument. input.ToKubeconfigPath must be an existing file when calling Move") - logDir := path.Join(input.LogFolder, "logs", input.Namespace) - Expect(os.MkdirAll(logDir, 0750)).To(Succeed(), "Invalid argument. input.LogFolder can't be created for Move") - - By("Moving workload clusters") - log.Logf("clusterctl move --from-kubeconfig %s --to-kubeconfig %s --namespace %s", - input.FromKubeconfigPath, - input.ToKubeconfigPath, - input.Namespace, - ) - - clusterctlClient, log := getClusterctlClientWithLogger(input.ClusterctlConfigPath, "clusterctl-move.log", logDir) - defer log.Close() - options := clusterctlclient.MoveOptions{ - FromKubeconfig: clusterctlclient.Kubeconfig{Path: input.FromKubeconfigPath, Context: ""}, - ToKubeconfig: clusterctlclient.Kubeconfig{Path: input.ToKubeconfigPath, Context: ""}, - Namespace: input.Namespace, - } - - Expect(clusterctlClient.Move(options)).To(Succeed(), "Failed to run clusterctl move") -} - -func getClusterctlClientWithLogger(configPath, logName, logFolder string) (clusterctlclient.Client, *logger.LogFile) { - log := logger.OpenLogFile(logger.OpenLogFileInput{ - LogFolder: logFolder, - Name: logName, - }) - clusterctllog.SetLogger(log.Logger()) - - c, err := clusterctlclient.New(configPath) - Expect(err).ToNot(HaveOccurred(), "Failed to create the clusterctl client library") - return c, log -} - -func valueOrDefault(v string) string { - if v != "" { - return v - } - return "(default)" -} diff --git a/test/e2e/framework/clusterctl/clusterctl_config.go b/test/e2e/framework/clusterctl/clusterctl_config.go deleted file mode 100644 index 96b6d233..00000000 --- a/test/e2e/framework/clusterctl/clusterctl_config.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package clusterctl - -import ( - "os" - - . "github.com/onsi/gomega" - "sigs.k8s.io/yaml" -) - -// Provide helpers for working with the clusterctl config file. - -// clusterctlConfig defines the content of the clusterctl config file. -// The main responsibility for this structure is to point clusterctl to the local repository that should be used for E2E tests. -type clusterctlConfig struct { - Path string - Values map[string]interface{} -} - -// providerConfig mirrors the clusterctl config.Provider interface and allows serialization of the corresponding info into a clusterctl config file. -type providerConfig struct { - Name string `json:"name,omitempty"` - URL string `json:"url,omitempty"` - Type string `json:"type,omitempty"` -} - -// write writes a clusterctl config file to disk. -func (c *clusterctlConfig) write() { - data, err := yaml.Marshal(c.Values) - Expect(err).ToNot(HaveOccurred(), "Failed to marshal the clusterctl config file") - - Expect(os.WriteFile(c.Path, data, 0600)).To(Succeed(), "Failed to write the clusterctl config file") -} - -// read reads a clusterctl config file from disk. -func (c *clusterctlConfig) read() { - data, err := os.ReadFile(c.Path) - Expect(err).ToNot(HaveOccurred()) - - err = yaml.Unmarshal(data, &c.Values) - Expect(err).ToNot(HaveOccurred(), "Failed to unmarshal the clusterctl config file") -} diff --git a/test/e2e/framework/clusterctl/clusterctl_helpers.go b/test/e2e/framework/clusterctl/clusterctl_helpers.go deleted file mode 100644 index 36d01586..00000000 --- a/test/e2e/framework/clusterctl/clusterctl_helpers.go +++ /dev/null @@ -1,385 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package clusterctl - -import ( - "context" - "os" - "path/filepath" - "time" - - . "github.com/onsi/gomega" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/cmd/clusterctl/client/config" - expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" - - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" - "github.com/kubesphere/kubekey/v3/test/e2e/framework" - "github.com/kubesphere/kubekey/v3/test/e2e/framework/internal/log" -) - -// InitManagementClusterAndWatchControllerLogsInput is the input type for InitManagementClusterAndWatchControllerLogs. -type InitManagementClusterAndWatchControllerLogsInput struct { - ClusterProxy framework.ClusterProxy - ClusterctlConfigPath string - CoreProvider string - BootstrapProviders []string - ControlPlaneProviders []string - InfrastructureProviders []string - LogFolder string - DisableMetricsCollection bool - ClusterctlBinaryPath string -} - -// InitManagementClusterAndWatchControllerLogs initializes a management using clusterctl and setup watches for controller logs. -// Important: Considering we want to support test suites using existing clusters, clusterctl init is executed only in case -// there are no provider controllers in the cluster; but controller logs watchers are created regardless of the pre-existing providers. -func InitManagementClusterAndWatchControllerLogs(ctx context.Context, input InitManagementClusterAndWatchControllerLogsInput, intervals ...interface{}) { - Expect(ctx).NotTo(BeNil(), "ctx is required for InitManagementClusterAndWatchControllerLogs") - Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling InitManagementClusterAndWatchControllerLogs") - Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling InitManagementClusterAndWatchControllerLogs") - Expect(input.InfrastructureProviders).ToNot(BeEmpty(), "Invalid argument. input.InfrastructureProviders can't be empty when calling InitManagementClusterAndWatchControllerLogs") - Expect(os.MkdirAll(input.LogFolder, 0750)).To(Succeed(), "Invalid argument. input.LogFolder can't be created for InitManagementClusterAndWatchControllerLogs") - - if input.CoreProvider == "" { - input.CoreProvider = config.ClusterAPIProviderName - } - if len(input.BootstrapProviders) == 0 { - input.BootstrapProviders = []string{K3sBootstrapProviderName} - } - if len(input.ControlPlaneProviders) == 0 { - input.ControlPlaneProviders = []string{K3sControlPlaneProviderName} - } - - client := input.ClusterProxy.GetClient() - controllersDeployments := framework.GetControllerDeployments(ctx, framework.GetControllerDeploymentsInput{ - Lister: client, - }) - if len(controllersDeployments) == 0 { - initInput := InitInput{ - // pass reference to the management cluster hosting this test - KubeconfigPath: input.ClusterProxy.GetKubeconfigPath(), - // pass the clusterctl config file that points to the local provider repository created for this test - ClusterctlConfigPath: input.ClusterctlConfigPath, - // setup the desired list of providers for a single-tenant management cluster - CoreProvider: input.CoreProvider, - BootstrapProviders: input.BootstrapProviders, - ControlPlaneProviders: input.ControlPlaneProviders, - InfrastructureProviders: input.InfrastructureProviders, - // setup clusterctl logs folder - LogFolder: input.LogFolder, - } - - if input.ClusterctlBinaryPath != "" { - InitWithBinary(ctx, input.ClusterctlBinaryPath, initInput) - } else { - Init(ctx, initInput) - } - } - - log.Logf("Waiting for provider controllers to be running") - controllersDeployments = framework.GetControllerDeployments(ctx, framework.GetControllerDeploymentsInput{ - Lister: client, - }) - Expect(controllersDeployments).ToNot(BeEmpty(), "The list of controller deployments should not be empty") - for _, deployment := range controllersDeployments { - framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ - Getter: client, - Deployment: deployment, - }, intervals...) - - // Start streaming logs from all controller providers - framework.WatchDeploymentLogs(ctx, framework.WatchDeploymentLogsInput{ - GetLister: client, - ClientSet: input.ClusterProxy.GetClientSet(), - Deployment: deployment, - LogPath: filepath.Join(input.LogFolder, "logs", deployment.GetNamespace()), - }) - - if !input.DisableMetricsCollection { - framework.WatchPodMetrics(ctx, framework.WatchPodMetricsInput{ - GetLister: client, - ClientSet: input.ClusterProxy.GetClientSet(), - Deployment: deployment, - MetricsPath: filepath.Join(input.LogFolder, "metrics", deployment.GetNamespace()), - }) - } - } -} - -// UpgradeManagementClusterAndWaitInput is the input type for UpgradeManagementClusterAndWait. -type UpgradeManagementClusterAndWaitInput struct { - ClusterProxy framework.ClusterProxy - ClusterctlConfigPath string - ClusterctlVariables map[string]string - Contract string - CoreProvider string - BootstrapProviders []string - ControlPlaneProviders []string - InfrastructureProviders []string - IPAMProviders []string - RuntimeExtensionProviders []string - LogFolder string -} - -// UpgradeManagementClusterAndWait upgrades provider a management cluster using clusterctl, and waits for the cluster to be ready. -func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagementClusterAndWaitInput, intervals ...interface{}) { - Expect(ctx).NotTo(BeNil(), "ctx is required for UpgradeManagementClusterAndWait") - Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling UpgradeManagementClusterAndWait") - Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling UpgradeManagementClusterAndWait") - // Check if the user want a custom upgrade - isCustomUpgrade := input.CoreProvider != "" || - len(input.BootstrapProviders) > 0 || - len(input.ControlPlaneProviders) > 0 || - len(input.InfrastructureProviders) > 0 || - len(input.IPAMProviders) > 0 || - len(input.RuntimeExtensionProviders) > 0 - - Expect((input.Contract != "" && !isCustomUpgrade) || (input.Contract == "" && isCustomUpgrade)).To(BeTrue(), `Invalid argument. Either the input.Contract parameter or at least one of the following providers has to be set: - input.CoreProvider, input.BootstrapProviders, input.ControlPlaneProviders, input.InfrastructureProviders, input.IPAMProviders, input.RuntimeExtensionProviders`) - - Expect(os.MkdirAll(input.LogFolder, 0750)).To(Succeed(), "Invalid argument. input.LogFolder can't be created for UpgradeManagementClusterAndWait") - - Upgrade(ctx, UpgradeInput{ - ClusterctlConfigPath: input.ClusterctlConfigPath, - ClusterctlVariables: input.ClusterctlVariables, - ClusterName: input.ClusterProxy.GetName(), - KubeconfigPath: input.ClusterProxy.GetKubeconfigPath(), - Contract: input.Contract, - CoreProvider: input.CoreProvider, - BootstrapProviders: input.BootstrapProviders, - ControlPlaneProviders: input.ControlPlaneProviders, - InfrastructureProviders: input.InfrastructureProviders, - IPAMProviders: input.IPAMProviders, - RuntimeExtensionProviders: input.RuntimeExtensionProviders, - LogFolder: input.LogFolder, - }) - - client := input.ClusterProxy.GetClient() - - log.Logf("Waiting for provider controllers to be running") - controllersDeployments := framework.GetControllerDeployments(ctx, framework.GetControllerDeploymentsInput{ - Lister: client, - ExcludeNamespaces: []string{"capi-webhook-system"}, // this namespace has been dropped in v1alpha4; this ensures we are not waiting for deployments being deleted as part of the upgrade process - }) - Expect(controllersDeployments).ToNot(BeEmpty(), "The list of controller deployments should not be empty") - for _, deployment := range controllersDeployments { - framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ - Getter: client, - Deployment: deployment, - }, intervals...) - - // Start streaming logs from all controller providers - framework.WatchDeploymentLogs(ctx, framework.WatchDeploymentLogsInput{ - GetLister: client, - ClientSet: input.ClusterProxy.GetClientSet(), - Deployment: deployment, - LogPath: filepath.Join(input.LogFolder, "logs", deployment.GetNamespace()), - }) - - framework.WatchPodMetrics(ctx, framework.WatchPodMetricsInput{ - GetLister: client, - ClientSet: input.ClusterProxy.GetClientSet(), - Deployment: deployment, - MetricsPath: filepath.Join(input.LogFolder, "metrics", deployment.GetNamespace()), - }) - } -} - -// ApplyClusterTemplateAndWaitInput is the input type for ApplyClusterTemplateAndWait. -type ApplyClusterTemplateAndWaitInput struct { - ClusterProxy framework.ClusterProxy - ConfigCluster ConfigClusterInput - CNIManifestPath string - WaitForClusterIntervals []interface{} - WaitForControlPlaneIntervals []interface{} - WaitForMachineDeployments []interface{} - WaitForMachinePools []interface{} - Args []string // extra args to be used during `kubectl apply` - PreWaitForCluster func() - PostMachinesProvisioned func() - ControlPlaneWaiters -} - -// Waiter is a function that runs and waits for a long-running operation to finish and updates the result. -type Waiter func(ctx context.Context, input ApplyClusterTemplateAndWaitInput, result *ApplyClusterTemplateAndWaitResult) - -// ControlPlaneWaiters are Waiter functions for the control plane. -type ControlPlaneWaiters struct { - WaitForControlPlaneInitialized Waiter - WaitForControlPlaneMachinesReady Waiter -} - -// ApplyClusterTemplateAndWaitResult is the output type for ApplyClusterTemplateAndWait. -type ApplyClusterTemplateAndWaitResult struct { - ClusterClass *clusterv1.ClusterClass - Cluster *clusterv1.Cluster - ControlPlane *infracontrolplanev1.K3sControlPlane - MachineDeployments []*clusterv1.MachineDeployment - MachinePools []*expv1.MachinePool -} - -// ExpectedWorkerNodes returns the expected number of worker nodes that will -// be provisioned by the given cluster template. -func (r *ApplyClusterTemplateAndWaitResult) ExpectedWorkerNodes() int32 { - expectedWorkerNodes := int32(0) - - for _, md := range r.MachineDeployments { - if md.Spec.Replicas != nil { - expectedWorkerNodes += *md.Spec.Replicas - } - } - for _, mp := range r.MachinePools { - if mp.Spec.Replicas != nil { - expectedWorkerNodes += *mp.Spec.Replicas - } - } - - return expectedWorkerNodes -} - -// ExpectedTotalNodes returns the expected number of nodes that will -// be provisioned by the given cluster template. -func (r *ApplyClusterTemplateAndWaitResult) ExpectedTotalNodes() int32 { - expectedNodes := r.ExpectedWorkerNodes() - - if r.ControlPlane != nil && r.ControlPlane.Spec.Replicas != nil { - expectedNodes += *r.ControlPlane.Spec.Replicas - } - - return expectedNodes -} - -// ApplyClusterTemplateAndWait gets a cluster template using clusterctl, and waits for the cluster to be ready. -// Important! this method assumes the cluster uses a K3sControlPlane and MachineDeployments. -func ApplyClusterTemplateAndWait(ctx context.Context, input ApplyClusterTemplateAndWaitInput, result *ApplyClusterTemplateAndWaitResult) { - setDefaults(&input) - Expect(ctx).NotTo(BeNil(), "ctx is required for ApplyClusterTemplateAndWait") - Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling ApplyClusterTemplateAndWait") - Expect(result).ToNot(BeNil(), "Invalid argument. result can't be nil when calling ApplyClusterTemplateAndWait") - Expect(input.ConfigCluster.ControlPlaneMachineCount).ToNot(BeNil()) - Expect(input.ConfigCluster.WorkerMachineCount).ToNot(BeNil()) - - log.Logf("Creating the workload cluster with name %q using the %q template (Kubernetes %s, %d control-plane machines, %d worker machines)", - input.ConfigCluster.ClusterName, valueOrDefault(input.ConfigCluster.Flavor), input.ConfigCluster.KubernetesVersion, *input.ConfigCluster.ControlPlaneMachineCount, *input.ConfigCluster.WorkerMachineCount) - - log.Logf("Getting the cluster template yaml") - workloadClusterTemplate := ConfigCluster(ctx, ConfigClusterInput{ - // pass reference to the management cluster hosting this test - KubeconfigPath: input.ConfigCluster.KubeconfigPath, - // pass the clusterctl config file that points to the local provider repository created for this test, - ClusterctlConfigPath: input.ConfigCluster.ClusterctlConfigPath, - // select template - Flavor: input.ConfigCluster.Flavor, - // define template variables - Namespace: input.ConfigCluster.Namespace, - ClusterName: input.ConfigCluster.ClusterName, - KubernetesVersion: input.ConfigCluster.KubernetesVersion, - ControlPlaneMachineCount: input.ConfigCluster.ControlPlaneMachineCount, - WorkerMachineCount: input.ConfigCluster.WorkerMachineCount, - InfrastructureProvider: input.ConfigCluster.InfrastructureProvider, - // setup clusterctl logs folder - LogFolder: input.ConfigCluster.LogFolder, - ClusterctlVariables: input.ConfigCluster.ClusterctlVariables, - }) - Expect(workloadClusterTemplate).ToNot(BeNil(), "Failed to get the cluster template") - - log.Logf("Applying the cluster template yaml to the cluster") - Eventually(func() error { - return input.ClusterProxy.Apply(ctx, workloadClusterTemplate, input.Args...) - }, 10*time.Second).Should(Succeed(), "Failed to apply the cluster template") - - // Once we applied the cluster template we can run PreWaitForCluster. - // Note: This can e.g. be used to verify the BeforeClusterCreate lifecycle hook is executed - // and blocking correctly. - if input.PreWaitForCluster != nil { - log.Logf("Calling PreWaitForCluster") - input.PreWaitForCluster() - } - - log.Logf("Waiting for the cluster infrastructure to be provisioned") - result.Cluster = framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{ - Getter: input.ClusterProxy.GetClient(), - Namespace: input.ConfigCluster.Namespace, - Name: input.ConfigCluster.ClusterName, - }, input.WaitForClusterIntervals...) - - if result.Cluster.Spec.Topology != nil { - result.ClusterClass = framework.GetClusterClassByName(ctx, framework.GetClusterClassByNameInput{ - Getter: input.ClusterProxy.GetClient(), - Namespace: input.ConfigCluster.Namespace, - Name: result.Cluster.Spec.Topology.Class, - }) - } - - log.Logf("Waiting for control plane to be initialized") - input.WaitForControlPlaneInitialized(ctx, input, result) - - if input.CNIManifestPath != "" { - log.Logf("Installing a CNI plugin to the workload cluster") - workloadCluster := input.ClusterProxy.GetWorkloadCluster(ctx, result.Cluster.Namespace, result.Cluster.Name) - - cniYaml, err := os.ReadFile(input.CNIManifestPath) - Expect(err).ShouldNot(HaveOccurred()) - - Expect(workloadCluster.Apply(ctx, cniYaml)).ShouldNot(HaveOccurred()) - } - - log.Logf("Waiting for control plane to be ready") - input.WaitForControlPlaneMachinesReady(ctx, input, result) - - log.Logf("Waiting for the machine deployments to be provisioned") - result.MachineDeployments = framework.DiscoveryAndWaitForMachineDeployments(ctx, framework.DiscoveryAndWaitForMachineDeploymentsInput{ - Lister: input.ClusterProxy.GetClient(), - Cluster: result.Cluster, - }, input.WaitForMachineDeployments...) - - log.Logf("Waiting for the machine pools to be provisioned") - result.MachinePools = framework.DiscoveryAndWaitForMachinePools(ctx, framework.DiscoveryAndWaitForMachinePoolsInput{ - Getter: input.ClusterProxy.GetClient(), - Lister: input.ClusterProxy.GetClient(), - Cluster: result.Cluster, - }, input.WaitForMachinePools...) - - if input.PostMachinesProvisioned != nil { - log.Logf("Calling PostMachinesProvisioned") - input.PostMachinesProvisioned() - } -} - -// setDefaults sets the default values for ApplyClusterTemplateAndWaitInput if not set. -// Currently, we set the default ControlPlaneWaiters here, which are implemented for K3sControlPlane. -func setDefaults(input *ApplyClusterTemplateAndWaitInput) { - if input.WaitForControlPlaneInitialized == nil { - input.WaitForControlPlaneInitialized = func(ctx context.Context, input ApplyClusterTemplateAndWaitInput, result *ApplyClusterTemplateAndWaitResult) { - result.ControlPlane = framework.DiscoveryAndWaitForControlPlaneInitialized(ctx, framework.DiscoveryAndWaitForControlPlaneInitializedInput{ - Lister: input.ClusterProxy.GetClient(), - Cluster: result.Cluster, - }, input.WaitForControlPlaneIntervals...) - } - } - - if input.WaitForControlPlaneMachinesReady == nil { - input.WaitForControlPlaneMachinesReady = func(ctx context.Context, input ApplyClusterTemplateAndWaitInput, result *ApplyClusterTemplateAndWaitResult) { - framework.WaitForControlPlaneAndMachinesReady(ctx, framework.WaitForControlPlaneAndMachinesReadyInput{ - GetLister: input.ClusterProxy.GetClient(), - Cluster: result.Cluster, - ControlPlane: result.ControlPlane, - }, input.WaitForControlPlaneIntervals...) - } - } -} diff --git a/test/e2e/framework/clusterctl/doc.go b/test/e2e/framework/clusterctl/doc.go deleted file mode 100644 index b7133a82..00000000 --- a/test/e2e/framework/clusterctl/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package clusterctl implements clusterctl interaction. -package clusterctl diff --git a/test/e2e/framework/clusterctl/e2e_config.go b/test/e2e/framework/clusterctl/e2e_config.go deleted file mode 100644 index 6b3c64b2..00000000 --- a/test/e2e/framework/clusterctl/e2e_config.go +++ /dev/null @@ -1,646 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package clusterctl - -import ( - "context" - "fmt" - "net/url" - "os" - "path/filepath" - "regexp" - "runtime" - "sort" - "strconv" - "strings" - "time" - - . "github.com/onsi/gomega" - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/utils/pointer" - clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" - clusterctlconfig "sigs.k8s.io/cluster-api/cmd/clusterctl/client/config" - "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/yaml" -) - -// GetVariable returns the value of a variable defined in the e2e config file. -const ( - K3sBootstrapProviderName = "k3s" - K3sControlPlaneProviderName = "k3s" -) - -// Provides access to the configuration for an e2e test. - -// LoadE2EConfigInput is the input for LoadE2EConfig. -type LoadE2EConfigInput struct { - // ConfigPath for the e2e test. - ConfigPath string -} - -// LoadE2EConfig loads the configuration for the e2e test environment. -func LoadE2EConfig(ctx context.Context, input LoadE2EConfigInput) *E2EConfig { - configData, err := os.ReadFile(input.ConfigPath) - Expect(err).ToNot(HaveOccurred(), "Failed to read the e2e test config file") - Expect(configData).ToNot(BeEmpty(), "The e2e test config file should not be empty") - - config := &E2EConfig{} - Expect(yaml.Unmarshal(configData, config)).To(Succeed(), "Failed to convert the e2e test config file to yaml") - - config.Defaults() - config.AbsPaths(filepath.Dir(input.ConfigPath)) - - Expect(config.Validate()).To(Succeed(), "The e2e test config file is not valid") - - return config -} - -// E2EConfig defines the configuration of an e2e test environment. -type E2EConfig struct { - // Name is the name of the Kind management cluster. - // Defaults to test-[random generated suffix]. - ManagementClusterName string `json:"managementClusterName,omitempty"` - - // Images is a list of container images to load into the Kind cluster. - Images []ContainerImage `json:"images,omitempty"` - - // Providers is a list of providers to be configured in the local repository that will be created for the e2e test. - // It is required to provide following providers - // - cluster-api - // - bootstrap kubeadm - // - control-plane kubeadm - // - one infrastructure provider - // The test will adapt to the selected infrastructure provider - Providers []ProviderConfig `json:"providers,omitempty"` - - // Variables to be added to the clusterctl config file - // Please note that clusterctl read variables from OS environment variables as well, so you can avoid to hard code - // sensitive data in the config file. - Variables map[string]string `json:"variables,omitempty"` - - // Intervals to be used for long operations during tests - Intervals map[string][]string `json:"intervals,omitempty"` -} - -// ProviderConfig describes a provider to be configured in the local repository that will be created for the e2e test. -type ProviderConfig struct { - // Name is the name of the provider. - Name string `json:"name"` - - // Type is the type of the provider. - Type string `json:"type"` - - // Versions is a list of component YAML to be added to the local repository, one for each release. - // Please note that the first source will be used a default release for this provider. - Versions []ProviderVersionSource `json:"versions,omitempty"` - - // Files is a list of files to be copied into the local repository for all the releases. - Files []Files `json:"files,omitempty"` -} - -// LoadImageBehavior indicates the behavior when loading an image. -type LoadImageBehavior string - -const ( - // MustLoadImage causes a load operation to fail if the image cannot be - // loaded. - MustLoadImage LoadImageBehavior = "mustLoad" - - // TryLoadImage causes any errors that occur when loading an image to be - // ignored. - TryLoadImage LoadImageBehavior = "tryLoad" -) - -// ContainerImage describes an image to load into a cluster and the behavior -// when loading the image. -type ContainerImage struct { - // Name is the fully qualified name of the image. - Name string - - // LoadBehavior may be used to dictate whether a failed load operation - // should fail the test run. This is useful when wanting to load images - // *if* they exist locally, but not wanting to fail if they don't. - // - // Defaults to MustLoadImage. - LoadBehavior LoadImageBehavior -} - -// ComponentSourceType indicates how a component's source should be obtained. -type ComponentSourceType string - -const ( - // URLSource is component YAML available directly via a URL. - // The URL may begin with http://, https:// or file://(can be omitted, relative paths supported). - URLSource ComponentSourceType = "url" - - // KustomizeSource is a valid kustomization root that can be used to produce - // the component YAML. - KustomizeSource ComponentSourceType = "kustomize" -) - -// ProviderVersionSource describes how to obtain a component's YAML. -type ProviderVersionSource struct { - // Name is used for logging when a component has multiple sources. - Name string `json:"name,omitempty"` - - // Value is the source of the component's YAML. - // May be a URL or a kustomization root (specified by Type). - // If a Type=url then Value may begin with file://, http://, or https://. - // If a Type=kustomize then Value may be any valid go-getter URL. For - // more information please see https://github.com/hashicorp/go-getter#url-format. - Value string `json:"value"` - - // Contract defines the Cluster API contract version a specific version of the provider abides to. - Contract string `json:"contract,omitempty"` - - // Type describes how to process the source of the component's YAML. - // - // Defaults to "kustomize". - Type ComponentSourceType `json:"type,omitempty"` - - // Replacements is a list of patterns to replace in the component YAML - // prior to application. - Replacements []ComponentReplacement `json:"replacements,omitempty"` - - // Files is a list of files to be copied into the local repository for this release. - Files []Files `json:"files,omitempty"` -} - -// ComponentWaiterType indicates the type of check to use to determine if the -// installed components are ready. -type ComponentWaiterType string - -const ( - // ServiceWaiter indicates to wait until a service's condition is Available. - // When ComponentWaiter.Value is set to "service", the ComponentWaiter.Value - // should be set to the name of a Service resource. - ServiceWaiter ComponentWaiterType = "service" - - // PodsWaiter indicates to wait until all the pods in a namespace have a - // condition of Ready. - // When ComponentWaiter.Value is set to "pods", the ComponentWaiter.Value - // should be set to the name of a Namespace resource. - PodsWaiter ComponentWaiterType = "pods" -) - -// ComponentWaiter contains information to help determine whether installed -// components are ready. -type ComponentWaiter struct { - // Value varies depending on the specified Type. - // Please see the documentation for the different WaiterType constants to - // understand the valid values for this field. - Value string `json:"value"` - - // Type describes the type of check to perform. - // - // Defaults to "pods". - Type ComponentWaiterType `json:"type,omitempty"` -} - -// ComponentReplacement is used to replace some of the generated YAML prior -// to application. -type ComponentReplacement struct { - // Old is the pattern to replace. - // A regular expression may be used. - Old string `json:"old"` - // New is the string used to replace the old pattern. - // An empty string is valid. - New string `json:"new,omitempty"` -} - -// ComponentConfig describes a component required by the e2e test environment. -type ComponentConfig struct { - // Name is the name of the component. - // This field is primarily used for logging. - Name string `json:"name"` - - // Sources is an optional list of component YAML to apply to the management - // cluster. - // This field may be omitted when wanting only to block progress via one or - // more Waiters. - Sources []ProviderVersionSource `json:"sources,omitempty"` - - // Waiters is an optional list of checks to perform in order to determine - // whether or not the installed components are ready. - Waiters []ComponentWaiter `json:"waiters,omitempty"` -} - -// Files contains information about files to be copied into the local repository. -type Files struct { - // SourcePath path of the file. - SourcePath string `json:"sourcePath"` - - // TargetName name of the file copied into the local repository. if empty, the source name - // Will be preserved - TargetName string `json:"targetName,omitempty"` -} - -// Defaults assigns default values to the object. More specifically: -// - ManagementClusterName gets a default name if empty. -// - Providers version gets type KustomizeSource if not otherwise specified. -// - Providers file gets targetName = sourceName if not otherwise specified. -// - Images gets LoadBehavior = MustLoadImage if not otherwise specified. -func (c *E2EConfig) Defaults() { - if c.ManagementClusterName == "" { - c.ManagementClusterName = fmt.Sprintf("test-%s", util.RandomString(6)) - } - for i := range c.Providers { - provider := &c.Providers[i] - for j := range provider.Versions { - version := &provider.Versions[j] - if version.Type == "" { - version.Type = KustomizeSource - } - for j := range version.Files { - file := &version.Files[j] - if file.SourcePath != "" && file.TargetName == "" { - file.TargetName = filepath.Base(file.SourcePath) - } - } - } - for j := range provider.Files { - file := &provider.Files[j] - if file.SourcePath != "" && file.TargetName == "" { - file.TargetName = filepath.Base(file.SourcePath) - } - } - } - imageReplacer := strings.NewReplacer("{OS}", runtime.GOOS, "{ARCH}", runtime.GOARCH) - for i := range c.Images { - containerImage := &c.Images[i] - containerImage.Name = imageReplacer.Replace(containerImage.Name) - if containerImage.LoadBehavior == "" { - containerImage.LoadBehavior = MustLoadImage - } - } -} - -// AbsPaths makes relative paths absolute using the given base path. -func (c *E2EConfig) AbsPaths(basePath string) { - for i := range c.Providers { - provider := &c.Providers[i] - for j := range provider.Versions { - version := &provider.Versions[j] - if version.Type != URLSource && version.Value != "" { - if !filepath.IsAbs(version.Value) { - version.Value = filepath.Join(basePath, version.Value) - } - } else if version.Type == URLSource && version.Value != "" { - // Skip error, will be checked later when loading contents from URL - u, _ := url.Parse(version.Value) - - if u != nil { - switch u.Scheme { - case "", fileURIScheme: - fp := strings.TrimPrefix(version.Value, fmt.Sprintf("%s://", fileURIScheme)) - if !filepath.IsAbs(fp) { - version.Value = filepath.Join(basePath, fp) - } - } - } - } - - for j := range version.Files { - file := &version.Files[j] - if file.SourcePath != "" { - if !filepath.IsAbs(file.SourcePath) { - file.SourcePath = filepath.Join(basePath, file.SourcePath) - } - } - } - } - for j := range provider.Files { - file := &provider.Files[j] - if file.SourcePath != "" { - if !filepath.IsAbs(file.SourcePath) { - file.SourcePath = filepath.Join(basePath, file.SourcePath) - } - } - } - } -} - -func errInvalidArg(format string, args ...interface{}) error { - msg := fmt.Sprintf(format, args...) - return errors.Errorf("invalid argument: %s", msg) -} - -func errEmptyArg(argName string) error { - return errInvalidArg("%s is empty", argName) -} - -// Validate validates the configuration. More specifically: -// - ManagementClusterName should not be empty. -// - There should be one CoreProvider (cluster-api), one BootstrapProvider (kubeadm), one ControlPlaneProvider (kubeadm). -// - There should be one InfraProvider (pick your own). -// - Image should have name and loadBehavior be one of [mustload, tryload]. -// - Intervals should be valid ginkgo intervals. -func (c *E2EConfig) Validate() error { - // ManagementClusterName should not be empty. - if c.ManagementClusterName == "" { - return errEmptyArg("ManagementClusterName") - } - - if err := c.validateProviders(); err != nil { - return err - } - - // Image should have name and loadBehavior be one of [mustload, tryload]. - for i, containerImage := range c.Images { - if containerImage.Name == "" { - return errEmptyArg(fmt.Sprintf("Images[%d].Name=%q", i, containerImage.Name)) - } - switch containerImage.LoadBehavior { - case MustLoadImage, TryLoadImage: - // Valid - default: - return errInvalidArg("Images[%d].LoadBehavior=%q", i, containerImage.LoadBehavior) - } - } - - // Intervals should be valid ginkgo intervals. - for k, intervals := range c.Intervals { - switch len(intervals) { - case 0: - return errInvalidArg("Intervals[%s]=%q", k, intervals) - case 1, 2: - default: - return errInvalidArg("Intervals[%s]=%q", k, intervals) - } - for _, i := range intervals { - if _, err := time.ParseDuration(i); err != nil { - return errInvalidArg("Intervals[%s]=%q", k, intervals) - } - } - } - return nil -} - -// validateProviders validates the provider configuration. More specifically: -// - Providers name should not be empty. -// - Providers type should be one of [CoreProvider, BootstrapProvider, ControlPlaneProvider, InfrastructureProvider]. -// - Providers version should have a name. -// - Providers version.type should be one of [url, kustomize]. -// - Providers version.replacements.old should be a valid regex. -// - Providers files should be an existing file and have a target name. -func (c *E2EConfig) validateProviders() error { - providersByType := map[clusterctlv1.ProviderType][]string{ - clusterctlv1.CoreProviderType: nil, - clusterctlv1.BootstrapProviderType: nil, - clusterctlv1.ControlPlaneProviderType: nil, - clusterctlv1.InfrastructureProviderType: nil, - } - for i, providerConfig := range c.Providers { - // Providers name should not be empty. - if providerConfig.Name == "" { - return errEmptyArg(fmt.Sprintf("Providers[%d].Name", i)) - } - // Providers type should be one of the know types. - providerType := clusterctlv1.ProviderType(providerConfig.Type) - switch providerType { - case clusterctlv1.CoreProviderType, clusterctlv1.BootstrapProviderType, clusterctlv1.ControlPlaneProviderType, clusterctlv1.InfrastructureProviderType: - providersByType[providerType] = append(providersByType[providerType], providerConfig.Name) - default: - return errInvalidArg("Providers[%d].Type=%q", i, providerConfig.Type) - } - - // Providers providerVersion should have a name. - // Providers providerVersion.type should be one of [url, kustomize]. - // Providers providerVersion.replacements.old should be a valid regex. - for j, providerVersion := range providerConfig.Versions { - if providerVersion.Name == "" { - return errEmptyArg(fmt.Sprintf("Providers[%d].Sources[%d].Name", i, j)) - } - if _, err := version.ParseSemantic(providerVersion.Name); err != nil { - return errInvalidArg("Providers[%d].Sources[%d].Name=%q", i, j, providerVersion.Name) - } - switch providerVersion.Type { - case URLSource, KustomizeSource: - if providerVersion.Value == "" { - return errEmptyArg(fmt.Sprintf("Providers[%d].Sources[%d].Value", i, j)) - } - default: - return errInvalidArg("Providers[%d].Sources[%d].Type=%q", i, j, providerVersion.Type) - } - for k, replacement := range providerVersion.Replacements { - if _, err := regexp.Compile(replacement.Old); err != nil { - return errInvalidArg("Providers[%d].Sources[%d].Replacements[%d].Old=%q: %v", i, j, k, replacement.Old, err) - } - } - // Providers files should be an existing file and have a target name. - for k, file := range providerVersion.Files { - if file.SourcePath == "" { - return errInvalidArg("Providers[%d].Sources[%d].Files[%d].SourcePath=%q", i, j, k, file.SourcePath) - } - if !fileExists(file.SourcePath) { - return errInvalidArg("Providers[%d].Sources[%d].Files[%d].SourcePath=%q", i, j, k, file.SourcePath) - } - if file.TargetName == "" { - return errInvalidArg("Providers[%d].Sources[%d].Files[%d].TargetName=%q", i, j, k, file.TargetName) - } - } - } - - // Providers files should be an existing file and have a target name. - for j, file := range providerConfig.Files { - if file.SourcePath == "" { - return errInvalidArg("Providers[%d].Files[%d].SourcePath=%q", i, j, file.SourcePath) - } - if !fileExists(file.SourcePath) { - return errInvalidArg("Providers[%d].Files[%d].SourcePath=%q", i, j, file.SourcePath) - } - if file.TargetName == "" { - return errInvalidArg("Providers[%d].Files[%d].TargetName=%q", i, j, file.TargetName) - } - } - } - - // There should be one CoreProvider (cluster-api), one BootstrapProvider (kubeadm), one ControlPlaneProvider (kubeadm). - if len(providersByType[clusterctlv1.CoreProviderType]) != 1 { - return errInvalidArg("invalid config: it is required to have exactly one core-provider") - } - if providersByType[clusterctlv1.CoreProviderType][0] != clusterctlconfig.ClusterAPIProviderName { - return errInvalidArg("invalid config: core-provider should be named %s", clusterctlconfig.ClusterAPIProviderName) - } - - if len(providersByType[clusterctlv1.BootstrapProviderType]) != 1 { - return errInvalidArg("invalid config: it is required to have exactly one bootstrap-provider") - } - if providersByType[clusterctlv1.BootstrapProviderType][0] != K3sBootstrapProviderName { - return errInvalidArg("invalid config: bootstrap-provider should be named %s", K3sBootstrapProviderName) - } - - if len(providersByType[clusterctlv1.ControlPlaneProviderType]) != 1 { - return errInvalidArg("invalid config: it is required to have exactly one control-plane-provider") - } - if providersByType[clusterctlv1.ControlPlaneProviderType][0] != K3sControlPlaneProviderName { - return errInvalidArg("invalid config: control-plane-provider should be named %s", K3sControlPlaneProviderName) - } - - // There should be one InfraProvider (pick your own). - if len(providersByType[clusterctlv1.InfrastructureProviderType]) < 1 { - return errInvalidArg("invalid config: it is required to have at least one infrastructure-provider") - } - return nil -} - -func fileExists(filename string) bool { - info, err := os.Stat(filename) - if os.IsNotExist(err) { - return false - } - return !info.IsDir() -} - -// InfrastructureProviders returns the infrastructure provider selected for running this E2E test. -func (c *E2EConfig) InfrastructureProviders() []string { - return c.getProviders(clusterctlv1.InfrastructureProviderType) -} - -func (c *E2EConfig) getProviders(t clusterctlv1.ProviderType) []string { - InfraProviders := []string{} - for _, provider := range c.Providers { - if provider.Type == string(t) { - InfraProviders = append(InfraProviders, provider.Name) - } - } - return InfraProviders -} - -// HasDockerProvider returns true if the Docker provider is configured for running this E2E test. -func (c *E2EConfig) HasDockerProvider() bool { - for _, i := range c.InfrastructureProviders() { - if i == "docker" { - return true - } - } - return false -} - -// GetIntervals returns the intervals to be applied to a Eventually operation. -// It searches for [spec]/[key] intervals first, and if it is not found, it searches -// for default/[key]. If also the default/[key] intervals are not found, -// ginkgo DefaultEventuallyTimeout and DefaultEventuallyPollingInterval are used. -func (c *E2EConfig) GetIntervals(spec, key string) []interface{} { - intervals, ok := c.Intervals[fmt.Sprintf("%s/%s", spec, key)] - if !ok { - if intervals, ok = c.Intervals[fmt.Sprintf("default/%s", key)]; !ok { - return nil - } - } - intervalsInterfaces := make([]interface{}, len(intervals)) - for i := range intervals { - intervalsInterfaces[i] = intervals[i] - } - return intervalsInterfaces -} - -// HasVariable returns true if the variable exists in the config. -func (c *E2EConfig) HasVariable(varName string) bool { - if _, ok := os.LookupEnv(varName); ok { - return true - } - - _, ok := c.Variables[varName] - return ok -} - -// GetVariable returns a variable from environment variables or from the e2e config file. -func (c *E2EConfig) GetVariable(varName string) string { - if value, ok := os.LookupEnv(varName); ok { - return value - } - - value, ok := c.Variables[varName] - Expect(ok).NotTo(BeFalse()) - return value -} - -// GetInt64PtrVariable returns an Int64Ptr variable from the e2e config file. -func (c *E2EConfig) GetInt64PtrVariable(varName string) *int64 { - wCountStr := c.GetVariable(varName) - if wCountStr == "" { - return nil - } - - wCount, err := strconv.ParseInt(wCountStr, 10, 64) - Expect(err).NotTo(HaveOccurred()) - return pointer.Int64(wCount) -} - -// GetInt32PtrVariable returns an Int32Ptr variable from the e2e config file. -func (c *E2EConfig) GetInt32PtrVariable(varName string) *int32 { - wCountStr := c.GetVariable(varName) - if wCountStr == "" { - return nil - } - - wCount, err := strconv.ParseUint(wCountStr, 10, 32) - Expect(err).NotTo(HaveOccurred()) - return pointer.Int32(int32(wCount)) -} - -// GetProviderVersions returns the sorted list of versions defined for a provider. -func (c *E2EConfig) GetProviderVersions(provider string) []string { - return c.getVersions(provider, "*") -} - -// GetProvidersWithOldestVersion returns the list of providers with the oldest version. -func (c *E2EConfig) GetProvidersWithOldestVersion(providers ...string) []string { - ret := make([]string, 0, len(providers)) - for _, p := range providers { - versions := c.getVersions(p, "*") - if len(versions) > 0 { - ret = append(ret, fmt.Sprintf("%s:%s", p, versions[0])) - } - } - return ret -} - -// GetProviderLatestVersionsByContract returns the latest version for each contract defined for a provider. -func (c *E2EConfig) GetProviderLatestVersionsByContract(contract string, providers ...string) []string { - ret := make([]string, 0, len(providers)) - for _, p := range providers { - versions := c.getVersions(p, contract) - if len(versions) > 0 { - ret = append(ret, fmt.Sprintf("%s:%s", p, versions[len(versions)-1])) - } - } - return ret -} - -func (c *E2EConfig) getVersions(provider string, contract string) []string { - versions := []string{} - for _, p := range c.Providers { - if p.Name == provider { - for _, v := range p.Versions { - if contract == "*" || v.Contract == contract { - versions = append(versions, v.Name) - } - } - } - } - - sort.Slice(versions, func(i, j int) bool { - // NOTE: Ignoring errors because the validity of the format is ensured by Validation. - vI, _ := version.ParseSemantic(versions[i]) - vJ, _ := version.ParseSemantic(versions[j]) - return vI.LessThan(vJ) - }) - return versions -} diff --git a/test/e2e/framework/clusterctl/logger/log_file.go b/test/e2e/framework/clusterctl/logger/log_file.go deleted file mode 100644 index 5f67e600..00000000 --- a/test/e2e/framework/clusterctl/logger/log_file.go +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package logger - -import ( - "bufio" - "os" - "path/filepath" - - "github.com/go-logr/logr" - . "github.com/onsi/gomega" -) - -// Provides a log_file that can be used to store the logs generated by clusterctl actions. - -// OpenLogFileInput is the input for OpenLogFile. -type OpenLogFileInput struct { - LogFolder string - Name string -} - -// OpenLogFile opens a new log file. -func OpenLogFile(input OpenLogFileInput) *LogFile { - filePath := filepath.Join(input.LogFolder, input.Name) - Expect(os.MkdirAll(filepath.Dir(filePath), 0750)).To(Succeed(), "Failed to create log folder %s", filepath.Dir(filePath)) - - f, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) //nolint:gosec // No security issue: filepath is safe. - Expect(err).ToNot(HaveOccurred(), "Failed to create log file %s", filePath) - - return &LogFile{ - name: input.Name, - file: f, - Writer: bufio.NewWriter(f), - } -} - -// LogFile is a log file. -type LogFile struct { - name string - file *os.File - *bufio.Writer -} - -// Name returns the name of the log file. -func (f *LogFile) Name() string { - return f.name -} - -// Flush flushes the log file. -func (f *LogFile) Flush() { - Expect(f.Writer.Flush()).To(Succeed(), "Failed to flush log %s", f.name) -} - -// Close closes the log file. -func (f *LogFile) Close() { - f.Flush() - Expect(f.file.Close()).To(Succeed(), "Failed to close log %s", f.name) -} - -// Logger returns a logger that writes to the log file. -func (f *LogFile) Logger() logr.Logger { - return logr.New(&logger{writer: f}) -} diff --git a/test/e2e/framework/clusterctl/logger/logger.go b/test/e2e/framework/clusterctl/logger/logger.go deleted file mode 100644 index 857152af..00000000 --- a/test/e2e/framework/clusterctl/logger/logger.go +++ /dev/null @@ -1,148 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package logger implements clusterctl logging functionality. -package logger - -import ( - "encoding/json" - "fmt" - "io" - - "github.com/go-logr/logr" - "github.com/pkg/errors" -) - -// Provides a logr.Logger to use during e2e tests. - -type logger struct { - writer io.Writer - values []interface{} -} - -var _ logr.LogSink = &logger{} - -func (l *logger) Init(info logr.RuntimeInfo) { -} - -func (l *logger) Enabled(level int) bool { - return true -} - -func (l *logger) Info(level int, msg string, kvs ...interface{}) { - values := copySlice(l.values) - values = append(values, kvs...) - values = append(values, "msg", msg) - f, err := flatten(values) - if err != nil { - panic(err) - } - fmt.Fprintln(l.writer, f) -} - -func (l *logger) Error(err error, msg string, kvs ...interface{}) { - panic("using log.Error is deprecated in clusterctl") -} - -func (l *logger) V(level int) logr.LogSink { - nl := l.clone() - return nl -} - -func (l *logger) WithName(name string) logr.LogSink { - panic("using log.WithName is deprecated in clusterctl") -} - -func (l *logger) WithValues(kvList ...interface{}) logr.LogSink { - nl := l.clone() - nl.values = append(nl.values, kvList...) - return nl -} - -func (l *logger) clone() *logger { - return &logger{ - writer: l.writer, - values: copySlice(l.values), - } -} - -func copySlice(in []interface{}) []interface{} { - out := make([]interface{}, len(in)) - copy(out, in) - return out -} - -func flatten(values []interface{}) (string, error) { - var msgValue string - var errorValue error - if len(values)%2 == 1 { - return "", errors.New("log entry cannot have odd number off keyAndValues") - } - - keys := make([]string, 0, len(values)/2) - val := make(map[string]interface{}, len(values)/2) - for i := 0; i < len(values); i += 2 { - k, ok := values[i].(string) - if !ok { - panic(fmt.Sprintf("key is not a string: %s", values[i])) - } - var v interface{} - if i+1 < len(values) { - v = values[i+1] - } - switch k { - case "msg": - msgValue, ok = v.(string) - if !ok { - panic(fmt.Sprintf("the msg value is not of type string: %s", v)) - } - case "error": - errorValue, ok = v.(error) - if !ok { - panic(fmt.Sprintf("the error value is not of type error: %s", v)) - } - default: - if _, ok := val[k]; !ok { - keys = append(keys, k) - } - val[k] = v - } - } - str := "" - str += msgValue - if errorValue != nil { - if msgValue != "" { - str += ": " - } - str += errorValue.Error() - } - for _, k := range keys { - prettyValue, err := pretty(val[k]) - if err != nil { - return "", err - } - str += fmt.Sprintf(" %s=%s", k, prettyValue) - } - return str, nil -} - -func pretty(value interface{}) (string, error) { - jb, err := json.Marshal(value) - if err != nil { - return "", errors.Wrapf(err, "Failed to marshal %s", value) - } - return string(jb), nil -} diff --git a/test/e2e/framework/clusterctl/repository.go b/test/e2e/framework/clusterctl/repository.go deleted file mode 100644 index 08712f0f..00000000 --- a/test/e2e/framework/clusterctl/repository.go +++ /dev/null @@ -1,284 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package clusterctl - -import ( - "bytes" - "context" - "fmt" - "io" - "net/http" - "net/url" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/blang/semver" - . "github.com/onsi/gomega" - "github.com/pkg/errors" - clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" - "sigs.k8s.io/cluster-api/test/framework/exec" - . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" -) - -const ( - fileURIScheme = "file" - httpURIScheme = "http" - httpsURIScheme = "https" -) - -// RepositoryFileTransformation is a helpers for managing a clusterctl -// local repository to be used for running e2e tests in isolation. -type RepositoryFileTransformation func([]byte) ([]byte, error) - -// CreateRepositoryInput is the input for CreateRepository. -type CreateRepositoryInput struct { - RepositoryFolder string - E2EConfig *E2EConfig - FileTransformations []RepositoryFileTransformation -} - -// RegisterClusterResourceSetConfigMapTransformation registers a FileTransformations that injects a manifests file into -// a ConfigMap that defines a ClusterResourceSet resource. -// -// NOTE: this transformation is specifically designed for replacing "data: ${envSubstVar}". -func (i *CreateRepositoryInput) RegisterClusterResourceSetConfigMapTransformation(manifestPath, envSubstVar string) { - Byf("Reading the ClusterResourceSet manifest %s", manifestPath) - manifestData, err := os.ReadFile(manifestPath) //nolint:gosec - Expect(err).ToNot(HaveOccurred(), "Failed to read the ClusterResourceSet manifest file") - Expect(manifestData).ToNot(BeEmpty(), "ClusterResourceSet manifest file should not be empty") - - i.FileTransformations = append(i.FileTransformations, func(template []byte) ([]byte, error) { - oldData := fmt.Sprintf("data: ${%s}", envSubstVar) - newData := "data:\n" - newData += " resources: |\n" - for _, l := range strings.Split(string(manifestData), "\n") { - newData += strings.Repeat(" ", 4) + l + "\n" - } - return bytes.ReplaceAll(template, []byte(oldData), []byte(newData)), nil - }) -} - -const clusterctlConfigFileName = "clusterctl-config.yaml" -const clusterctlConfigV1_2FileName = "clusterctl-config.v1.2.yaml" - -// CreateRepository creates a clusterctl local repository based on the e2e test config, and the returns the path -// to a clusterctl config file to be used for working with such repository. -func CreateRepository(ctx context.Context, input CreateRepositoryInput) string { - Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling CreateRepository") - Expect(os.MkdirAll(input.RepositoryFolder, 0750)).To(Succeed(), "Failed to create the clusterctl local repository folder %s", input.RepositoryFolder) - - providers := []providerConfig{} - providersV1_2 := []providerConfig{} - for _, provider := range input.E2EConfig.Providers { - providerLabel := clusterctlv1.ManifestLabel(provider.Name, clusterctlv1.ProviderType(provider.Type)) - providerURL := filepath.Join(input.RepositoryFolder, providerLabel, "latest", "components.yaml") - for _, version := range provider.Versions { - manifest, err := YAMLForComponentSource(ctx, version) - Expect(err).ToNot(HaveOccurred(), "Failed to generate the manifest for %q / %q", providerLabel, version.Name) - - sourcePath := filepath.Join(input.RepositoryFolder, providerLabel, version.Name) - Expect(os.MkdirAll(sourcePath, 0750)).To(Succeed(), "Failed to create the clusterctl local repository folder for %q / %q", providerLabel, version.Name) - - filePath := filepath.Join(sourcePath, "components.yaml") - Expect(os.WriteFile(filePath, manifest, 0600)).To(Succeed(), "Failed to write manifest in the clusterctl local repository for %q / %q", providerLabel, version.Name) - - destinationPath := filepath.Join(input.RepositoryFolder, providerLabel, version.Name, "components.yaml") - allFiles := append(provider.Files, version.Files...) - for _, file := range allFiles { - data, err := os.ReadFile(file.SourcePath) - Expect(err).ToNot(HaveOccurred(), "Failed to read file %q / %q", provider.Name, file.SourcePath) - - // Applies FileTransformations if defined - for _, t := range input.FileTransformations { - data, err = t(data) - Expect(err).ToNot(HaveOccurred(), "Failed to apply transformation func template %q", file) - } - - destinationFile := filepath.Join(filepath.Dir(destinationPath), file.TargetName) - Expect(os.WriteFile(destinationFile, data, 0600)).To(Succeed(), "Failed to write clusterctl local repository file %q / %q", provider.Name, file.TargetName) - } - } - p := providerConfig{ - Name: provider.Name, - URL: providerURL, - Type: provider.Type, - } - providers = append(providers, p) - } - - // set this path to an empty file under the repository path, so test can run in isolation without user's overrides kicking in - overridePath := filepath.Join(input.RepositoryFolder, "overrides") - Expect(os.MkdirAll(overridePath, 0750)).To(Succeed(), "Failed to create the clusterctl overrides folder %q", overridePath) - - // creates a clusterctl config file to be used for working with such repository - clusterctlConfigFile := &clusterctlConfig{ - Path: filepath.Join(input.RepositoryFolder, clusterctlConfigFileName), - Values: map[string]interface{}{ - "providers": providers, - "overridesFolder": overridePath, - }, - } - for key := range input.E2EConfig.Variables { - clusterctlConfigFile.Values[key] = input.E2EConfig.GetVariable(key) - } - clusterctlConfigFile.write() - - // creates a clusterctl config file to be used for working with such repository with only the providers supported in clusterctl < v1.3 - clusterctlConfigFileV1_2 := &clusterctlConfig{ - Path: filepath.Join(input.RepositoryFolder, clusterctlConfigV1_2FileName), - Values: map[string]interface{}{ - "providers": providersV1_2, - "overridesFolder": overridePath, - }, - } - for key := range input.E2EConfig.Variables { - clusterctlConfigFileV1_2.Values[key] = input.E2EConfig.GetVariable(key) - } - clusterctlConfigFileV1_2.write() - - return clusterctlConfigFile.Path -} - -// copyAndAmendClusterctlConfigInput is the input for copyAndAmendClusterctlConfig. -type copyAndAmendClusterctlConfigInput struct { - ClusterctlConfigPath string - OutputPath string - Variables map[string]string -} - -// copyAndAmendClusterctlConfig copies the clusterctl-config from ClusterctlConfigPath to -// OutputPath and adds the given Variables. -func copyAndAmendClusterctlConfig(_ context.Context, input copyAndAmendClusterctlConfigInput) { - // Read clusterctl config from ClusterctlConfigPath. - clusterctlConfigFile := &clusterctlConfig{ - Path: input.ClusterctlConfigPath, - } - clusterctlConfigFile.read() - - // Overwrite variables. - if clusterctlConfigFile.Values == nil { - clusterctlConfigFile.Values = map[string]interface{}{} - } - for key, value := range input.Variables { - clusterctlConfigFile.Values[key] = value - } - - // Write clusterctl config to OutputPath. - clusterctlConfigFile.Path = input.OutputPath - clusterctlConfigFile.write() -} - -// AdjustConfigPathForBinary adjusts the clusterctlConfigPath in case the clusterctl version v1.3. -func AdjustConfigPathForBinary(clusterctPath, clusterctlConfigPath string) string { - clusterctl := exec.NewCommand( - exec.WithCommand(clusterctPath), - exec.WithArgs("version", "--output", "short"), - ) - stdout, stderr, err := clusterctl.Run(context.Background()) - if err != nil { - Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl version:\nstdout:\n%s\nstderr:\n%s", string(stdout), string(stderr)) - } - data := stdout - version, err := semver.ParseTolerant(string(data)) - if err != nil { - Expect(err).ToNot(HaveOccurred(), "clusterctl version returned an invalid version: %s", string(data)) - } - - if version.LT(semver.MustParse("1.3.0")) { - return strings.Replace(clusterctlConfigPath, clusterctlConfigFileName, clusterctlConfigV1_2FileName, -1) - } - return clusterctlConfigPath -} - -// YAMLForComponentSource returns the YAML for the provided component source. -func YAMLForComponentSource(ctx context.Context, source ProviderVersionSource) ([]byte, error) { - var data []byte - - switch source.Type { - case URLSource: - buf, err := getComponentSourceFromURL(ctx, source) - if err != nil { - return nil, errors.Wrap(err, "failed to get component source YAML from URL") - } - data = buf - case KustomizeSource: - // Set Path of kustomize binary using CAPI_KUSTOMIZE_PATH env - kustomizePath, ok := os.LookupEnv("CAPI_KUSTOMIZE_PATH") - if !ok { - kustomizePath = "kustomize" - } - kustomize := exec.NewCommand( - exec.WithCommand(kustomizePath), - exec.WithArgs("build", source.Value)) - stdout, stderr, err := kustomize.Run(ctx) - if err != nil { - return nil, errors.Wrapf(err, "failed to execute kustomize: %s", stderr) - } - data = stdout - default: - return nil, errors.Errorf("invalid type: %q", source.Type) - } - - for _, replacement := range source.Replacements { - rx, err := regexp.Compile(replacement.Old) - if err != nil { - return nil, err - } - data = rx.ReplaceAll(data, []byte(replacement.New)) - } - - return data, nil -} - -// getComponentSourceFromURL fetches contents of component source YAML file from provided URL source. -func getComponentSourceFromURL(ctx context.Context, source ProviderVersionSource) ([]byte, error) { - var buf []byte - - u, err := url.Parse(source.Value) - if err != nil { - return nil, err - } - - // url.Parse always lower cases scheme - switch u.Scheme { - case "", fileURIScheme: - buf, err = os.ReadFile(u.Path) - if err != nil { - return nil, errors.Wrap(err, "failed to read file") - } - case httpURIScheme, httpsURIScheme: - req, err := http.NewRequestWithContext(ctx, http.MethodGet, source.Value, http.NoBody) - if err != nil { - return nil, errors.Wrapf(err, "failed to get %s: failed to create request", source.Value) - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, errors.Wrapf(err, "failed to get %s", source.Value) - } - defer resp.Body.Close() - buf, err = io.ReadAll(resp.Body) - if err != nil { - return nil, errors.Wrapf(err, "failed to get %s: failed to read body", source.Value) - } - default: - return nil, errors.Errorf("unknown scheme for component source %q: allowed values are file, http, https", u.Scheme) - } - - return buf, nil -} diff --git a/test/e2e/framework/controller_helpers.go b/test/e2e/framework/controller_helpers.go deleted file mode 100644 index a4327a37..00000000 --- a/test/e2e/framework/controller_helpers.go +++ /dev/null @@ -1,59 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "context" - - . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" -) - -// GetControllerDeploymentsInput is the input for GetControllerDeployments. -type GetControllerDeploymentsInput struct { - Lister Lister - ExcludeNamespaces []string -} - -// GetControllerDeployments returns all the deployment for the cluster API controllers existing in a management cluster. -func GetControllerDeployments(ctx context.Context, input GetControllerDeploymentsInput) []*appsv1.Deployment { - deploymentList := &appsv1.DeploymentList{} - Eventually(func() error { - return input.Lister.List(ctx, deploymentList, capiProviderOptions()...) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list deployments for the cluster API controllers") - - deployments := make([]*appsv1.Deployment, 0, len(deploymentList.Items)) - for i := range deploymentList.Items { - d := &deploymentList.Items[i] - if !skipDeployment(d, input.ExcludeNamespaces) { - deployments = append(deployments, d) - } - } - return deployments -} - -func skipDeployment(d *appsv1.Deployment, excludeNamespaces []string) bool { - if !d.DeletionTimestamp.IsZero() { - return true - } - for _, n := range excludeNamespaces { - if d.Namespace == n { - return true - } - } - return false -} diff --git a/test/e2e/framework/controlplane_helpers.go b/test/e2e/framework/controlplane_helpers.go deleted file mode 100644 index 03b3e0b6..00000000 --- a/test/e2e/framework/controlplane_helpers.go +++ /dev/null @@ -1,431 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/klog/v2" - "k8s.io/utils/pointer" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/util/patch" - "sigs.k8s.io/controller-runtime/pkg/client" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" - "github.com/kubesphere/kubekey/v3/test/e2e/framework/internal/log" -) - -// CreateK3sControlPlaneInput is the input for CreateK3sControlPlane. -type CreateK3sControlPlaneInput struct { - Creator Creator - ControlPlane *infracontrolplanev1.K3sControlPlane - MachineTemplate client.Object -} - -// CreateK3sControlPlane creates the control plane object and necessary dependencies. -func CreateK3sControlPlane(ctx context.Context, input CreateK3sControlPlaneInput, intervals ...interface{}) { - By("creating the machine template") - Eventually(func() error { - return input.Creator.Create(ctx, input.MachineTemplate) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to create MachineTemplate %s", input.MachineTemplate.GetName()) - - By("creating a K3sControlPlane") - Eventually(func() error { - err := input.Creator.Create(ctx, input.ControlPlane) - if err != nil { - log.Logf("Failed to create the K3sControlPlane: %+v", err) - } - return err - }, intervals...).Should(Succeed(), "Failed to create the K3sControlPlane %s", klog.KObj(input.ControlPlane)) -} - -// GetK3sControlPlaneByClusterInput is the input for GetK3sControlPlaneByCluster. -type GetK3sControlPlaneByClusterInput struct { - Lister Lister - ClusterName string - Namespace string -} - -// GetK3sControlPlaneByCluster returns the K3sControlPlane objects for a cluster. -// Important! this method relies on labels that are created by the CAPI controllers during the first reconciliation, so -// it is necessary to ensure this is already happened before calling it. -func GetK3sControlPlaneByCluster(ctx context.Context, input GetK3sControlPlaneByClusterInput) *infracontrolplanev1.K3sControlPlane { - controlPlaneList := &infracontrolplanev1.K3sControlPlaneList{} - Eventually(func() error { - return input.Lister.List(ctx, controlPlaneList, byClusterOptions(input.ClusterName, input.Namespace)...) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list K3sControlPlane object for Cluster %s", klog.KRef(input.Namespace, input.ClusterName)) - Expect(len(controlPlaneList.Items)).ToNot(BeNumerically(">", 1), "Cluster %s should not have more than 1 K3sControlPlane object", klog.KRef(input.Namespace, input.ClusterName)) - if len(controlPlaneList.Items) == 1 { - return &controlPlaneList.Items[0] - } - return nil -} - -// WaitForK3sControlPlaneMachinesToExistInput is the input for WaitForK3sControlPlaneMachinesToExist. -type WaitForK3sControlPlaneMachinesToExistInput struct { - Lister Lister - Cluster *clusterv1.Cluster - ControlPlane *infracontrolplanev1.K3sControlPlane -} - -// WaitForK3sControlPlaneMachinesToExist will wait until all control plane machines have node refs. -func WaitForK3sControlPlaneMachinesToExist(ctx context.Context, input WaitForK3sControlPlaneMachinesToExistInput, intervals ...interface{}) { - By("Waiting for all control plane nodes to exist") - inClustersNamespaceListOption := client.InNamespace(input.Cluster.Namespace) - // ControlPlane labels - matchClusterListOption := client.MatchingLabels{ - clusterv1.MachineControlPlaneLabelName: "", - clusterv1.ClusterLabelName: input.Cluster.Name, - } - - Eventually(func() (int, error) { - machineList := &clusterv1.MachineList{} - if err := input.Lister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption); err != nil { - log.Logf("Failed to list the machines: %+v", err) - return 0, err - } - count := 0 - for _, machine := range machineList.Items { - if machine.Status.NodeRef != nil { - count++ - } - } - return count, nil - }, intervals...).Should(Equal(int(*input.ControlPlane.Spec.Replicas)), "Timed out waiting for %d control plane machines to exist", int(*input.ControlPlane.Spec.Replicas)) -} - -// WaitForOneK3sControlPlaneMachineToExistInput is the input for WaitForK3sControlPlaneMachinesToExist. -type WaitForOneK3sControlPlaneMachineToExistInput struct { - Lister Lister - Cluster *clusterv1.Cluster - ControlPlane *infracontrolplanev1.K3sControlPlane -} - -// WaitForOneK3sControlPlaneMachineToExist will wait until all control plane machines have node refs. -func WaitForOneK3sControlPlaneMachineToExist(ctx context.Context, input WaitForOneK3sControlPlaneMachineToExistInput, intervals ...interface{}) { - Expect(ctx).NotTo(BeNil(), "ctx is required for WaitForOneK3sControlPlaneMachineToExist") - Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Getter can't be nil when calling WaitForOneK3sControlPlaneMachineToExist") - Expect(input.ControlPlane).ToNot(BeNil(), "Invalid argument. input.ControlPlane can't be nil when calling WaitForOneK3sControlPlaneMachineToExist") - - By("Waiting for one control plane node to exist") - inClustersNamespaceListOption := client.InNamespace(input.Cluster.Namespace) - // ControlPlane labels - matchClusterListOption := client.MatchingLabels{ - clusterv1.MachineControlPlaneLabelName: "", - clusterv1.ClusterLabelName: input.Cluster.Name, - } - - Eventually(func() (bool, error) { - machineList := &clusterv1.MachineList{} - if err := input.Lister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption); err != nil { - log.Logf("Failed to list the machines: %+v", err) - return false, err - } - count := 0 - for _, machine := range machineList.Items { - if machine.Status.NodeRef != nil { - count++ - } - } - return count > 0, nil - }, intervals...).Should(BeTrue(), "No Control Plane machines came into existence. ") -} - -// WaitForControlPlaneToBeReadyInput is the input for WaitForControlPlaneToBeReady. -type WaitForControlPlaneToBeReadyInput struct { - Getter Getter - ControlPlane *infracontrolplanev1.K3sControlPlane -} - -// WaitForControlPlaneToBeReady will wait for a control plane to be ready. -func WaitForControlPlaneToBeReady(ctx context.Context, input WaitForControlPlaneToBeReadyInput, intervals ...interface{}) { - By("Waiting for the control plane to be ready") - controlplane := &infracontrolplanev1.K3sControlPlane{} - Eventually(func() (infracontrolplanev1.K3sControlPlane, error) { - key := client.ObjectKey{ - Namespace: input.ControlPlane.GetNamespace(), - Name: input.ControlPlane.GetName(), - } - if err := input.Getter.Get(ctx, key, controlplane); err != nil { - return *controlplane, errors.Wrapf(err, "failed to get KCP") - } - return *controlplane, nil - }, intervals...).Should(MatchFields(IgnoreExtras, Fields{ - "Status": MatchFields(IgnoreExtras, Fields{ - "Ready": BeTrue(), - }), - }), PrettyPrint(controlplane)+"\n") -} - -// AssertControlPlaneFailureDomainsInput is the input for AssertControlPlaneFailureDomains. -type AssertControlPlaneFailureDomainsInput struct { - Lister Lister - Cluster *clusterv1.Cluster -} - -// AssertControlPlaneFailureDomains will look at all control plane machines and see what failure domains they were -// placed in. If machines were placed in unexpected or wrong failure domains the expectation will fail. -func AssertControlPlaneFailureDomains(ctx context.Context, input AssertControlPlaneFailureDomainsInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for AssertControlPlaneFailureDomains") - Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Lister can't be nil when calling AssertControlPlaneFailureDomains") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling AssertControlPlaneFailureDomains") - - By("Checking all the control plane machines are in the expected failure domains") - controlPlaneFailureDomains := sets.NewString() - for fd, fdSettings := range input.Cluster.Status.FailureDomains { - if fdSettings.ControlPlane { - controlPlaneFailureDomains.Insert(fd) - } - } - - // Look up all the control plane machines. - inClustersNamespaceListOption := client.InNamespace(input.Cluster.Namespace) - matchClusterListOption := client.MatchingLabels{ - clusterv1.ClusterLabelName: input.Cluster.Name, - clusterv1.MachineControlPlaneLabelName: "", - } - - machineList := &clusterv1.MachineList{} - Eventually(func() error { - return input.Lister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Couldn't list control-plane machines for the cluster %q", input.Cluster.Name) - - for _, machine := range machineList.Items { - if machine.Spec.FailureDomain != nil { - machineFD := *machine.Spec.FailureDomain - if !controlPlaneFailureDomains.Has(machineFD) { - Fail(fmt.Sprintf("Machine %s is in the %q failure domain, expecting one of the failure domain defined at cluster level", machine.Name, machineFD)) - } - } - } -} - -// DiscoveryAndWaitForControlPlaneInitializedInput is the input type for DiscoveryAndWaitForControlPlaneInitialized. -type DiscoveryAndWaitForControlPlaneInitializedInput struct { - Lister Lister - Cluster *clusterv1.Cluster -} - -// DiscoveryAndWaitForControlPlaneInitialized discovers the K3sControlPlane object attached to a cluster and waits for it to be initialized. -func DiscoveryAndWaitForControlPlaneInitialized(ctx context.Context, input DiscoveryAndWaitForControlPlaneInitializedInput, intervals ...interface{}) *infracontrolplanev1.K3sControlPlane { - Expect(ctx).NotTo(BeNil(), "ctx is required for DiscoveryAndWaitForControlPlaneInitialized") - Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Lister can't be nil when calling DiscoveryAndWaitForControlPlaneInitialized") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling DiscoveryAndWaitForControlPlaneInitialized") - - var controlPlane *infracontrolplanev1.K3sControlPlane - Eventually(func(g Gomega) { - controlPlane = GetK3sControlPlaneByCluster(ctx, GetK3sControlPlaneByClusterInput{ - Lister: input.Lister, - ClusterName: input.Cluster.Name, - Namespace: input.Cluster.Namespace, - }) - g.Expect(controlPlane).ToNot(BeNil()) - }, "10s", "1s").Should(Succeed(), "Couldn't get the control plane for the cluster %s", klog.KObj(input.Cluster)) - - log.Logf("Waiting for the first control plane machine managed by %s to be provisioned", klog.KObj(controlPlane)) - WaitForOneK3sControlPlaneMachineToExist(ctx, WaitForOneK3sControlPlaneMachineToExistInput{ - Lister: input.Lister, - Cluster: input.Cluster, - ControlPlane: controlPlane, - }, intervals...) - - return controlPlane -} - -// WaitForControlPlaneAndMachinesReadyInput is the input type for WaitForControlPlaneAndMachinesReady. -type WaitForControlPlaneAndMachinesReadyInput struct { - GetLister GetLister - Cluster *clusterv1.Cluster - ControlPlane *infracontrolplanev1.K3sControlPlane -} - -// WaitForControlPlaneAndMachinesReady waits for a K3sControlPlane object to be ready (all the machine provisioned and one node ready). -func WaitForControlPlaneAndMachinesReady(ctx context.Context, input WaitForControlPlaneAndMachinesReadyInput, intervals ...interface{}) { - Expect(ctx).NotTo(BeNil(), "ctx is required for WaitForControlPlaneReady") - Expect(input.GetLister).ToNot(BeNil(), "Invalid argument. input.GetLister can't be nil when calling WaitForControlPlaneReady") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling WaitForControlPlaneReady") - Expect(input.ControlPlane).ToNot(BeNil(), "Invalid argument. input.ControlPlane can't be nil when calling WaitForControlPlaneReady") - - if input.ControlPlane.Spec.Replicas != nil && int(*input.ControlPlane.Spec.Replicas) > 1 { - log.Logf("Waiting for the remaining control plane machines managed by %s to be provisioned", klog.KObj(input.ControlPlane)) - WaitForK3sControlPlaneMachinesToExist(ctx, WaitForK3sControlPlaneMachinesToExistInput{ - Lister: input.GetLister, - Cluster: input.Cluster, - ControlPlane: input.ControlPlane, - }, intervals...) - } - - log.Logf("Waiting for control plane %s to be ready (implies underlying nodes to be ready as well)", klog.KObj(input.ControlPlane)) - waitForControlPlaneToBeReadyInput := WaitForControlPlaneToBeReadyInput{ - Getter: input.GetLister, - ControlPlane: input.ControlPlane, - } - WaitForControlPlaneToBeReady(ctx, waitForControlPlaneToBeReadyInput, intervals...) - - AssertControlPlaneFailureDomains(ctx, AssertControlPlaneFailureDomainsInput{ - Lister: input.GetLister, - Cluster: input.Cluster, - }) -} - -// UpgradeControlPlaneAndWaitForUpgradeInput is the input type for UpgradeControlPlaneAndWaitForUpgrade. -type UpgradeControlPlaneAndWaitForUpgradeInput struct { - ClusterProxy ClusterProxy - Cluster *clusterv1.Cluster - ControlPlane *infracontrolplanev1.K3sControlPlane - KubernetesUpgradeVersion string - UpgradeMachineTemplate *string - EtcdImageTag string - DNSImageTag string - WaitForMachinesToBeUpgraded []interface{} - WaitForDNSUpgrade []interface{} - WaitForKubeProxyUpgrade []interface{} - WaitForEtcdUpgrade []interface{} -} - -// UpgradeControlPlaneAndWaitForUpgrade upgrades a K3sControlPlane and waits for it to be upgraded. -func UpgradeControlPlaneAndWaitForUpgrade(ctx context.Context, input UpgradeControlPlaneAndWaitForUpgradeInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for UpgradeControlPlaneAndWaitForUpgrade") - Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling UpgradeControlPlaneAndWaitForUpgrade") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling UpgradeControlPlaneAndWaitForUpgrade") - Expect(input.ControlPlane).ToNot(BeNil(), "Invalid argument. input.ControlPlane can't be nil when calling UpgradeControlPlaneAndWaitForUpgrade") - Expect(input.KubernetesUpgradeVersion).ToNot(BeNil(), "Invalid argument. input.KubernetesUpgradeVersion can't be empty when calling UpgradeControlPlaneAndWaitForUpgrade") - Expect(input.EtcdImageTag).ToNot(BeNil(), "Invalid argument. input.EtcdImageTag can't be empty when calling UpgradeControlPlaneAndWaitForUpgrade") - Expect(input.DNSImageTag).ToNot(BeNil(), "Invalid argument. input.DNSImageTag can't be empty when calling UpgradeControlPlaneAndWaitForUpgrade") - - mgmtClient := input.ClusterProxy.GetClient() - - log.Logf("Patching the new kubernetes version to KCP") - patchHelper, err := patch.NewHelper(input.ControlPlane, mgmtClient) - Expect(err).ToNot(HaveOccurred()) - - input.ControlPlane.Spec.Version = input.KubernetesUpgradeVersion - if input.UpgradeMachineTemplate != nil { - input.ControlPlane.Spec.MachineTemplate.InfrastructureRef.Name = *input.UpgradeMachineTemplate - } - // If the ClusterConfiguration is not specified, create an empty one. - if input.ControlPlane.Spec.K3sConfigSpec.ServerConfiguration == nil { - input.ControlPlane.Spec.K3sConfigSpec.ServerConfiguration = new(infrabootstrapv1.ServerConfiguration) - } - - if input.ControlPlane.Spec.K3sConfigSpec.ServerConfiguration.Database.ClusterInit == nil { - input.ControlPlane.Spec.K3sConfigSpec.ServerConfiguration.Database.ClusterInit = pointer.Bool(true) - } - - Eventually(func() error { - return patchHelper.Patch(ctx, input.ControlPlane) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to patch the new kubernetes version to KCP %s", klog.KObj(input.ControlPlane)) - - log.Logf("Waiting for control-plane machines to have the upgraded kubernetes version") - WaitForControlPlaneMachinesToBeUpgraded(ctx, WaitForControlPlaneMachinesToBeUpgradedInput{ - Lister: mgmtClient, - Cluster: input.Cluster, - MachineCount: int(*input.ControlPlane.Spec.Replicas), - KubernetesUpgradeVersion: input.KubernetesUpgradeVersion, - }, input.WaitForMachinesToBeUpgraded...) - - log.Logf("Waiting for kube-proxy to have the upgraded kubernetes version") - workloadCluster := input.ClusterProxy.GetWorkloadCluster(ctx, input.Cluster.Namespace, input.Cluster.Name) - workloadClient := workloadCluster.GetClient() - WaitForKubeProxyUpgrade(ctx, WaitForKubeProxyUpgradeInput{ - Getter: workloadClient, - KubernetesVersion: input.KubernetesUpgradeVersion, - }, input.WaitForKubeProxyUpgrade...) - - log.Logf("Waiting for CoreDNS to have the upgraded image tag") - WaitForDNSUpgrade(ctx, WaitForDNSUpgradeInput{ - Getter: workloadClient, - DNSVersion: input.DNSImageTag, - }, input.WaitForDNSUpgrade...) - - log.Logf("Waiting for etcd to have the upgraded image tag") - lblSelector, err := labels.Parse("component=etcd") - Expect(err).ToNot(HaveOccurred()) - WaitForPodListCondition(ctx, WaitForPodListConditionInput{ - Lister: workloadClient, - ListOptions: &client.ListOptions{LabelSelector: lblSelector}, - Condition: EtcdImageTagCondition(input.EtcdImageTag, int(*input.ControlPlane.Spec.Replicas)), - }, input.WaitForEtcdUpgrade...) -} - -// controlPlaneMachineOptions returns a set of ListOptions that allows to get all machine objects belonging to control plane. -func controlPlaneMachineOptions() []client.ListOption { - return []client.ListOption{ - client.HasLabels{clusterv1.MachineControlPlaneLabelName}, - } -} - -// ScaleAndWaitControlPlaneInput is the input for ScaleAndWaitControlPlane. -type ScaleAndWaitControlPlaneInput struct { - ClusterProxy ClusterProxy - Cluster *clusterv1.Cluster - ControlPlane *infracontrolplanev1.K3sControlPlane - Replicas int32 - WaitForControlPlane []interface{} -} - -// ScaleAndWaitControlPlane scales KCP and waits until all machines have node ref and equal to Replicas. -func ScaleAndWaitControlPlane(ctx context.Context, input ScaleAndWaitControlPlaneInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for ScaleAndWaitControlPlane") - Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling ScaleAndWaitControlPlane") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling ScaleAndWaitControlPlane") - - patchHelper, err := patch.NewHelper(input.ControlPlane, input.ClusterProxy.GetClient()) - Expect(err).ToNot(HaveOccurred()) - scaleBefore := pointer.Int32Deref(input.ControlPlane.Spec.Replicas, 0) - input.ControlPlane.Spec.Replicas = pointer.Int32(input.Replicas) - log.Logf("Scaling controlplane %s from %v to %v replicas", klog.KObj(input.ControlPlane), scaleBefore, input.Replicas) - Eventually(func() error { - return patchHelper.Patch(ctx, input.ControlPlane) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to scale controlplane %s from %v to %v replicas", klog.KObj(input.ControlPlane), scaleBefore, input.Replicas) - - log.Logf("Waiting for correct number of replicas to exist") - Eventually(func() (int, error) { - kcpLabelSelector, err := metav1.ParseToLabelSelector(input.ControlPlane.Status.Selector) - if err != nil { - return -1, err - } - - selector, err := metav1.LabelSelectorAsSelector(kcpLabelSelector) - if err != nil { - return -1, err - } - machines := &clusterv1.MachineList{} - if err := input.ClusterProxy.GetClient().List(ctx, machines, &client.ListOptions{LabelSelector: selector, Namespace: input.ControlPlane.Namespace}); err != nil { - return -1, err - } - nodeRefCount := 0 - for _, machine := range machines.Items { - if machine.Status.NodeRef != nil { - nodeRefCount++ - } - } - if len(machines.Items) != nodeRefCount { - return -1, errors.New("Machine count does not match existing nodes count") - } - return nodeRefCount, nil - }, input.WaitForControlPlane...).Should(Equal(int(input.Replicas)), "Timed out waiting for %d replicas to exist for control-plane %s", int(input.Replicas), klog.KObj(input.ControlPlane)) -} diff --git a/test/e2e/framework/convenience.go b/test/e2e/framework/convenience.go deleted file mode 100644 index d0f29924..00000000 --- a/test/e2e/framework/convenience.go +++ /dev/null @@ -1,88 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "reflect" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - apiextensionsv1beta "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - "k8s.io/apimachinery/pkg/runtime" - clusterv1alpha3 "sigs.k8s.io/cluster-api/api/v1alpha3" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" - addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1" - expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" - runtimev1 "sigs.k8s.io/cluster-api/exp/runtime/api/v1alpha1" - - infrabootstrapv1 "github.com/kubesphere/kubekey/v3/bootstrap/k3s/api/v1beta1" - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" -) - -// TryAddDefaultSchemes tries to add the following schemes: -// - Kubernetes corev1 -// - Kubernetes appsv1 -// - CAPI core -// - K3s Bootstrapper -// - K3s ControlPlane -// -// Any error that occurs when trying to add the schemes is ignored. -func TryAddDefaultSchemes(scheme *runtime.Scheme) { - // Add the core schemes. - _ = corev1.AddToScheme(scheme) - - // Add the apps schemes. - _ = appsv1.AddToScheme(scheme) - - // Add the core CAPI scheme. - _ = clusterv1.AddToScheme(scheme) - - // Add the CAPI experiments scheme. - _ = expv1.AddToScheme(scheme) - _ = addonsv1.AddToScheme(scheme) - - // Add the CAPI clusterctl scheme. - _ = clusterctlv1.AddToScheme(scheme) - - // Add the core CAPI v1alpha3 scheme. - _ = clusterv1alpha3.AddToScheme(scheme) - - // Add the k3s bootstrapper scheme. - _ = infrabootstrapv1.AddToScheme(scheme) - - // Add the k3s controlplane scheme. - _ = infracontrolplanev1.AddToScheme(scheme) - - // Add the api extensions (CRD) to the scheme. - _ = apiextensionsv1beta.AddToScheme(scheme) - _ = apiextensionsv1.AddToScheme(scheme) - - // Add RuntimeSDK to the scheme. - _ = runtimev1.AddToScheme(scheme) - - // Add rbac to the scheme. - _ = rbacv1.AddToScheme(scheme) -} - -// ObjectToKind returns the Kind without the package prefix. Pass in a pointer to a struct -// This will panic if used incorrectly. -func ObjectToKind(i runtime.Object) string { - return reflect.ValueOf(i).Elem().Type().Name() -} diff --git a/test/e2e/framework/daemonset_helpers.go b/test/e2e/framework/daemonset_helpers.go deleted file mode 100644 index 1d688b9f..00000000 --- a/test/e2e/framework/daemonset_helpers.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "context" - - "github.com/blang/semver" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - containerutil "sigs.k8s.io/cluster-api/util/container" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// WaitForKubeProxyUpgradeInput is the input for WaitForKubeProxyUpgrade. -type WaitForKubeProxyUpgradeInput struct { - Getter Getter - KubernetesVersion string -} - -// WaitForKubeProxyUpgrade waits until kube-proxy version matches with the kubernetes version. This is called during KCP upgrade. -func WaitForKubeProxyUpgrade(ctx context.Context, input WaitForKubeProxyUpgradeInput, intervals ...interface{}) { - By("Ensuring kube-proxy has the correct image") - - parsedVersion, err := semver.ParseTolerant(input.KubernetesVersion) - Expect(err).ToNot(HaveOccurred()) - - // Beginning with kubernetes v1.25, kubernetes images including kube-proxy get published to registry.k8s.io instead of k8s.gcr.io. - // This ensures that the imageRepository setting gets patched to registry.k8s.io when upgrading from v1.24 or lower, - // but only if there was no imageRespository explicitly set at the K3sControlPlanes ClusterConfiguration. - // This follows the behavior of `kubeadm upgrade`. - wantKubeProxyRegistry := "registry.k8s.io" - if parsedVersion.LT(semver.Version{Major: 1, Minor: 25, Patch: 0, Pre: []semver.PRVersion{{VersionStr: "alpha"}}}) { - wantKubeProxyRegistry = "k8s.gcr.io" - } - wantKubeProxyImage := wantKubeProxyRegistry + "/kube-proxy:" + containerutil.SemverToOCIImageTag(input.KubernetesVersion) - - Eventually(func() (bool, error) { - ds := &appsv1.DaemonSet{} - - if err := input.Getter.Get(ctx, client.ObjectKey{Name: "kube-proxy", Namespace: metav1.NamespaceSystem}, ds); err != nil { - return false, err - } - - if ds.Spec.Template.Spec.Containers[0].Image == wantKubeProxyImage { - return true, nil - } - return false, nil - }, intervals...).Should(BeTrue()) -} diff --git a/test/e2e/framework/deployment_helpers.go b/test/e2e/framework/deployment_helpers.go deleted file mode 100644 index 839dab5b..00000000 --- a/test/e2e/framework/deployment_helpers.go +++ /dev/null @@ -1,518 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "bufio" - "context" - "encoding/json" - "fmt" - "io" - "os" - "path" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - policyv1 "k8s.io/api/policy/v1" - "k8s.io/api/policy/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - utilversion "k8s.io/apimachinery/pkg/util/version" - "k8s.io/apimachinery/pkg/version" - "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - "k8s.io/utils/pointer" - . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" - "sigs.k8s.io/controller-runtime/pkg/client" - - infracontrolplanev1 "github.com/kubesphere/kubekey/v3/controlplane/k3s/api/v1beta1" - "github.com/kubesphere/kubekey/v3/test/e2e/framework/internal/log" -) - -const ( - nodeRoleOldControlPlane = "node-role.kubernetes.io/master" // Deprecated: https://github.com/kubernetes/kubeadm/issues/2200 - nodeRoleControlPlane = "node-role.kubernetes.io/control-plane" -) - -// WaitForDeploymentsAvailableInput is the input for WaitForDeploymentsAvailable. -type WaitForDeploymentsAvailableInput struct { - Getter Getter - Deployment *appsv1.Deployment -} - -// WaitForDeploymentsAvailable waits until the Deployment has status.Available = True, that signals that -// all the desired replicas are in place. -// This can be used to check if Cluster API controllers installed in the management cluster are working. -func WaitForDeploymentsAvailable(ctx context.Context, input WaitForDeploymentsAvailableInput, intervals ...interface{}) { - Byf("Waiting for deployment %s to be available", klog.KObj(input.Deployment)) - deployment := &appsv1.Deployment{} - Eventually(func() bool { - key := client.ObjectKey{ - Namespace: input.Deployment.GetNamespace(), - Name: input.Deployment.GetName(), - } - if err := input.Getter.Get(ctx, key, deployment); err != nil { - return false - } - for _, c := range deployment.Status.Conditions { - if c.Type == appsv1.DeploymentAvailable && c.Status == corev1.ConditionTrue { - return true - } - } - return false - }, intervals...).Should(BeTrue(), func() string { return DescribeFailedDeployment(input, deployment) }) -} - -// DescribeFailedDeployment returns detailed output to help debug a deployment failure in e2e. -func DescribeFailedDeployment(input WaitForDeploymentsAvailableInput, deployment *appsv1.Deployment) string { - b := strings.Builder{} - b.WriteString(fmt.Sprintf("Deployment %s failed to get status.Available = True condition", - klog.KObj(input.Deployment))) - if deployment == nil { - b.WriteString("\nDeployment: nil\n") - } else { - b.WriteString(fmt.Sprintf("\nDeployment:\n%s\n", PrettyPrint(deployment))) - } - return b.String() -} - -// WatchDeploymentLogsInput is the input for WatchDeploymentLogs. -type WatchDeploymentLogsInput struct { - GetLister GetLister - ClientSet *kubernetes.Clientset - Deployment *appsv1.Deployment - LogPath string -} - -// logMetadata contains metadata about the logs. -// The format is very similar to the one used by promtail. -type logMetadata struct { - Job string `json:"job"` - Namespace string `json:"namespace"` - App string `json:"app"` - Pod string `json:"pod"` - Container string `json:"container"` - NodeName string `json:"node_name"` - Stream string `json:"stream"` -} - -// WatchDeploymentLogs streams logs for all containers for all pods belonging to a deployment. Each container's logs are streamed -// in a separate goroutine so they can all be streamed concurrently. This only causes a test failure if there are errors -// retrieving the deployment, its pods, or setting up a log file. If there is an error with the log streaming itself, -// that does not cause the test to fail. -func WatchDeploymentLogs(ctx context.Context, input WatchDeploymentLogsInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for WatchControllerLogs") - Expect(input.ClientSet).NotTo(BeNil(), "input.ClientSet is required for WatchControllerLogs") - Expect(input.Deployment).NotTo(BeNil(), "input.Deployment is required for WatchControllerLogs") - - deployment := &appsv1.Deployment{} - key := client.ObjectKeyFromObject(input.Deployment) - Eventually(func() error { - return input.GetLister.Get(ctx, key, deployment) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get deployment %s", klog.KObj(input.Deployment)) - - selector, err := metav1.LabelSelectorAsMap(deployment.Spec.Selector) - Expect(err).NotTo(HaveOccurred(), "Failed to Pods selector for deployment %s", klog.KObj(input.Deployment)) - - pods := &corev1.PodList{} - Expect(input.GetLister.List(ctx, pods, client.InNamespace(input.Deployment.Namespace), client.MatchingLabels(selector))).To(Succeed(), "Failed to list Pods for deployment %s", klog.KObj(input.Deployment)) - - for _, pod := range pods.Items { - for _, container := range deployment.Spec.Template.Spec.Containers { - log.Logf("Creating log watcher for controller %s, pod %s, container %s", klog.KObj(input.Deployment), pod.Name, container.Name) - - // Create log metadata file. - logMetadataFile := filepath.Clean(path.Join(input.LogPath, input.Deployment.Name, pod.Name, container.Name+"-log-metadata.json")) - Expect(os.MkdirAll(filepath.Dir(logMetadataFile), 0750)).To(Succeed()) - - metadata := logMetadata{ - Job: input.Deployment.Namespace + "/" + input.Deployment.Name, - Namespace: input.Deployment.Namespace, - App: input.Deployment.Name, - Pod: pod.Name, - Container: container.Name, - NodeName: pod.Spec.NodeName, - Stream: "stderr", - } - metadataBytes, err := json.Marshal(&metadata) - Expect(err).To(BeNil()) - Expect(os.WriteFile(logMetadataFile, metadataBytes, 0600)).To(Succeed()) - - // Watch each container's logs in a goroutine so we can stream them all concurrently. - go func(pod corev1.Pod, container corev1.Container) { - defer GinkgoRecover() - - logFile := filepath.Clean(path.Join(input.LogPath, input.Deployment.Name, pod.Name, container.Name+".log")) - Expect(os.MkdirAll(filepath.Dir(logFile), 0750)).To(Succeed()) - - f, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) - Expect(err).NotTo(HaveOccurred()) - defer f.Close() - - opts := &corev1.PodLogOptions{ - Container: container.Name, - Follow: true, - } - - podLogs, err := input.ClientSet.CoreV1().Pods(input.Deployment.Namespace).GetLogs(pod.Name, opts).Stream(ctx) - if err != nil { - // Failing to stream logs should not cause the test to fail - log.Logf("Error starting logs stream for pod %s, container %s: %v", klog.KRef(pod.Namespace, pod.Name), container.Name, err) - return - } - defer podLogs.Close() - - out := bufio.NewWriter(f) - defer out.Flush() - _, err = out.ReadFrom(podLogs) - if err != nil && err != io.ErrUnexpectedEOF { - // Failing to stream logs should not cause the test to fail - log.Logf("Got error while streaming logs for pod %s, container %s: %v", klog.KRef(pod.Namespace, pod.Name), container.Name, err) - } - }(pod, container) - } - } -} - -// WatchPodMetricsInput is the input for WatchPodMetrics. -type WatchPodMetricsInput struct { - GetLister GetLister - ClientSet *kubernetes.Clientset - Deployment *appsv1.Deployment - MetricsPath string -} - -// WatchPodMetrics captures metrics from all pods every 5s. It expects to find port 8080 open on the controller. -func WatchPodMetrics(ctx context.Context, input WatchPodMetricsInput) { - // Dump machine metrics every 5 seconds - ticker := time.NewTicker(time.Second * 5) - Expect(ctx).NotTo(BeNil(), "ctx is required for dumpContainerMetrics") - Expect(input.ClientSet).NotTo(BeNil(), "input.ClientSet is required for dumpContainerMetrics") - Expect(input.Deployment).NotTo(BeNil(), "input.Deployment is required for dumpContainerMetrics") - - deployment := &appsv1.Deployment{} - key := client.ObjectKeyFromObject(input.Deployment) - Eventually(func() error { - return input.GetLister.Get(ctx, key, deployment) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get deployment %s", klog.KObj(input.Deployment)) - - selector, err := metav1.LabelSelectorAsMap(deployment.Spec.Selector) - Expect(err).NotTo(HaveOccurred(), "Failed to Pods selector for deployment %s", klog.KObj(input.Deployment)) - - pods := &corev1.PodList{} - Eventually(func() error { - return input.GetLister.List(ctx, pods, client.InNamespace(input.Deployment.Namespace), client.MatchingLabels(selector)) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list Pods for deployment %s", klog.KObj(input.Deployment)) - - go func() { - defer GinkgoRecover() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - dumpPodMetrics(ctx, input.ClientSet, input.MetricsPath, deployment.Name, pods) - } - } - }() -} - -// dumpPodMetrics captures metrics from all pods. It expects to find port 8080 open on the controller. -func dumpPodMetrics(ctx context.Context, client *kubernetes.Clientset, metricsPath string, deploymentName string, pods *corev1.PodList) { - for _, pod := range pods.Items { - metricsDir := path.Join(metricsPath, deploymentName, pod.Name) - metricsFile := path.Join(metricsDir, "metrics.txt") - Expect(os.MkdirAll(metricsDir, 0750)).To(Succeed()) - - res := client.CoreV1().RESTClient().Get(). - Namespace(pod.Namespace). - Resource("pods"). - Name(fmt.Sprintf("%s:8080", pod.Name)). - SubResource("proxy"). - Suffix("metrics"). - Do(ctx) - data, err := res.Raw() - - if err != nil { - // Failing to dump metrics should not cause the test to fail - data = []byte(fmt.Sprintf("Error retrieving metrics for pod %s: %v\n%s", klog.KRef(pod.Namespace, pod.Name), err, string(data))) - metricsFile = path.Join(metricsDir, "metrics-error.txt") - } - - if err := os.WriteFile(metricsFile, data, 0600); err != nil { - // Failing to dump metrics should not cause the test to fail - log.Logf("Error writing metrics for pod %s: %v", klog.KRef(pod.Namespace, pod.Name), err) - } - } -} - -// WaitForDNSUpgradeInput is the input for WaitForDNSUpgrade. -type WaitForDNSUpgradeInput struct { - Getter Getter - DNSVersion string -} - -// WaitForDNSUpgrade waits until CoreDNS version matches with the CoreDNS upgrade version and all its replicas -// are ready for use with the upgraded version. This is called during KCP upgrade. -func WaitForDNSUpgrade(ctx context.Context, input WaitForDNSUpgradeInput, intervals ...interface{}) { - By("Ensuring CoreDNS has the correct image") - - Eventually(func() (bool, error) { - d := &appsv1.Deployment{} - - if err := input.Getter.Get(ctx, client.ObjectKey{Name: "coredns", Namespace: metav1.NamespaceSystem}, d); err != nil { - return false, err - } - - // NOTE: coredns image name has changed over time (k8s.gcr.io/coredns, - // k8s.gcr.io/coredns/coredns), so we are checking if the version actually changed. - if strings.HasSuffix(d.Spec.Template.Spec.Containers[0].Image, fmt.Sprintf(":%s", input.DNSVersion)) { - return true, nil - } - - // check whether the upgraded CoreDNS replicas are available and ready for use. - if d.Status.ObservedGeneration >= d.Generation { - if d.Spec.Replicas != nil && d.Status.UpdatedReplicas == *d.Spec.Replicas && d.Status.AvailableReplicas == *d.Spec.Replicas { - return true, nil - } - } - - return false, nil - }, intervals...).Should(BeTrue()) -} - -// DeployUnevictablePodInput is the input for DeployUnevictablePod. -type DeployUnevictablePodInput struct { - WorkloadClusterProxy ClusterProxy - ControlPlane *infracontrolplanev1.K3sControlPlane - DeploymentName string - Namespace string - - WaitForDeploymentAvailableInterval []interface{} -} - -// DeployUnevictablePod deploys a pod that is not evictable. -func DeployUnevictablePod(ctx context.Context, input DeployUnevictablePodInput) { - Expect(input.DeploymentName).ToNot(BeNil(), "Need a deployment name in DeployUnevictablePod") - Expect(input.Namespace).ToNot(BeNil(), "Need a namespace in DeployUnevictablePod") - Expect(input.WorkloadClusterProxy).ToNot(BeNil(), "Need a workloadClusterProxy in DeployUnevictablePod") - - EnsureNamespace(ctx, input.WorkloadClusterProxy.GetClient(), input.Namespace) - - workloadDeployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: input.DeploymentName, - Namespace: input.Namespace, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: pointer.Int32(4), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "nonstop", - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": "nonstop", - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "web", - Image: "nginx:1.12", - Ports: []corev1.ContainerPort{ - { - Name: "http", - Protocol: corev1.ProtocolTCP, - ContainerPort: 80, - }, - }, - }, - }, - }, - }, - }, - } - workloadClient := input.WorkloadClusterProxy.GetClientSet() - - if input.ControlPlane != nil { - var serverVersion *version.Info - Eventually(func() error { - var err error - serverVersion, err = workloadClient.ServerVersion() - return err - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "failed to get server version") - - // Use the control-plane label for Kubernetes version >= v1.20.0. - if utilversion.MustParseGeneric(serverVersion.String()).AtLeast(utilversion.MustParseGeneric("v1.20.0")) { - workloadDeployment.Spec.Template.Spec.NodeSelector = map[string]string{nodeRoleControlPlane: ""} - } else { - workloadDeployment.Spec.Template.Spec.NodeSelector = map[string]string{nodeRoleOldControlPlane: ""} - } - - workloadDeployment.Spec.Template.Spec.Tolerations = []corev1.Toleration{ - { - Key: nodeRoleOldControlPlane, - Effect: "NoSchedule", - }, - { - Key: nodeRoleControlPlane, - Effect: "NoSchedule", - }, - } - } - AddDeploymentToWorkloadCluster(ctx, AddDeploymentToWorkloadClusterInput{ - Namespace: input.Namespace, - ClientSet: workloadClient, - Deployment: workloadDeployment, - }) - - // TODO(oscr): Remove when Kubernetes 1.20 support is dropped. - serverVersion, err := workloadClient.ServerVersion() - Expect(err).ToNot(HaveOccurred(), "Failed to get Kubernetes version for workload") - - // If Kubernetes < 1.21.0 we need to use PDB from v1beta1 - if utilversion.MustParseGeneric(serverVersion.String()).LessThan(utilversion.MustParseGeneric("v1.21.0")) { - budgetV1Beta1 := &v1beta1.PodDisruptionBudget{ - TypeMeta: metav1.TypeMeta{ - Kind: "PodDisruptionBudget", - APIVersion: "policy/v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: input.DeploymentName, - Namespace: input.Namespace, - }, - Spec: v1beta1.PodDisruptionBudgetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "nonstop", - }, - }, - MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Int, - IntVal: 1, - StrVal: "1", - }, - }, - } - - AddPodDisruptionBudgetV1Beta1(ctx, AddPodDisruptionBudgetInputV1Beta1{ - Namespace: input.Namespace, - ClientSet: workloadClient, - Budget: budgetV1Beta1, - }) - - // If Kubernetes >= 1.21.0 then we need to use PDB from v1 - } else { - budget := &policyv1.PodDisruptionBudget{ - TypeMeta: metav1.TypeMeta{ - Kind: "PodDisruptionBudget", - APIVersion: "policy/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: input.DeploymentName, - Namespace: input.Namespace, - }, - Spec: policyv1.PodDisruptionBudgetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "nonstop", - }, - }, - MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Int, - IntVal: 1, - StrVal: "1", - }, - }, - } - - AddPodDisruptionBudget(ctx, AddPodDisruptionBudgetInput{ - Namespace: input.Namespace, - ClientSet: workloadClient, - Budget: budget, - }) - } - - WaitForDeploymentsAvailable(ctx, WaitForDeploymentsAvailableInput{ - Getter: input.WorkloadClusterProxy.GetClient(), - Deployment: workloadDeployment, - }, input.WaitForDeploymentAvailableInterval...) -} - -// AddDeploymentToWorkloadClusterInput is the input for AddDeploymentToWorkloadCluster. -type AddDeploymentToWorkloadClusterInput struct { - ClientSet *kubernetes.Clientset - Deployment *appsv1.Deployment - Namespace string -} - -// AddDeploymentToWorkloadCluster adds a deployment to the workload cluster. -func AddDeploymentToWorkloadCluster(ctx context.Context, input AddDeploymentToWorkloadClusterInput) { - Eventually(func() error { - result, err := input.ClientSet.AppsV1().Deployments(input.Namespace).Create(ctx, input.Deployment, metav1.CreateOptions{}) - if result != nil && err == nil { - return nil - } - return fmt.Errorf("deployment %s not successfully created in workload cluster: %v", klog.KObj(input.Deployment), err) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to create deployment %s in workload cluster", klog.KObj(input.Deployment)) -} - -// AddPodDisruptionBudgetInput is the input for AddPodDisruptionBudget. -type AddPodDisruptionBudgetInput struct { - ClientSet *kubernetes.Clientset - Budget *policyv1.PodDisruptionBudget - Namespace string -} - -// AddPodDisruptionBudget adds a PodDisruptionBudget to the workload cluster. -func AddPodDisruptionBudget(ctx context.Context, input AddPodDisruptionBudgetInput) { - Eventually(func() error { - budget, err := input.ClientSet.PolicyV1().PodDisruptionBudgets(input.Namespace).Create(ctx, input.Budget, metav1.CreateOptions{}) - if budget != nil && err == nil { - return nil - } - return fmt.Errorf("podDisruptionBudget needs to be successfully deployed: %v", err) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "podDisruptionBudget needs to be successfully deployed") -} - -// TODO(oscr): Delete below when Kubernetes 1.20 support is dropped. - -// AddPodDisruptionBudgetInputV1Beta1 is the input for AddPodDisruptionBudgetV1Beta1. -type AddPodDisruptionBudgetInputV1Beta1 struct { - ClientSet *kubernetes.Clientset - Budget *v1beta1.PodDisruptionBudget - Namespace string -} - -// AddPodDisruptionBudgetV1Beta1 adds a PodDisruptionBudget to the workload cluster. -func AddPodDisruptionBudgetV1Beta1(ctx context.Context, input AddPodDisruptionBudgetInputV1Beta1) { - Eventually(func() error { - budget, err := input.ClientSet.PolicyV1beta1().PodDisruptionBudgets(input.Namespace).Create(ctx, input.Budget, metav1.CreateOptions{}) - if budget != nil && err == nil { - return nil - } - return fmt.Errorf("podDisruptionBudget needs to be successfully deployed: %v", err) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "podDisruptionBudget needs to be successfully deployed") -} diff --git a/test/e2e/framework/doc.go b/test/e2e/framework/doc.go deleted file mode 100644 index 3db4c896..00000000 --- a/test/e2e/framework/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package framework implements end to end testing. -package framework diff --git a/test/e2e/framework/interfaces.go b/test/e2e/framework/interfaces.go deleted file mode 100644 index 28d516f1..00000000 --- a/test/e2e/framework/interfaces.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "context" - - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// Interfaces to scope down client.Client. - -// Getter can get resources. -type Getter interface { - Get(ctx context.Context, key client.ObjectKey, obj client.Object) error -} - -// Creator can creates resources. -type Creator interface { - Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error -} - -// Lister can lists resources. -type Lister interface { - List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error -} - -// Deleter can delete resources. -type Deleter interface { - Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error -} - -// GetLister can get and list resources. -type GetLister interface { - Getter - Lister -} diff --git a/test/e2e/framework/internal/log/log.go b/test/e2e/framework/internal/log/log.go deleted file mode 100644 index 5d731534..00000000 --- a/test/e2e/framework/internal/log/log.go +++ /dev/null @@ -1,29 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package log implements test framework logging. -package log - -import ( - "fmt" - - . "github.com/onsi/ginkgo" -) - -// Logf logs a message at the specified log level. -func Logf(format string, a ...interface{}) { - fmt.Fprintf(GinkgoWriter, "INFO: "+format+"\n", a...) -} diff --git a/test/e2e/framework/machine_helpers.go b/test/e2e/framework/machine_helpers.go deleted file mode 100644 index 604a064c..00000000 --- a/test/e2e/framework/machine_helpers.go +++ /dev/null @@ -1,277 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "context" - - . "github.com/onsi/gomega" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/klog/v2" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" - "sigs.k8s.io/cluster-api/util/conditions" - "sigs.k8s.io/cluster-api/util/patch" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/kubesphere/kubekey/v3/test/e2e/framework/internal/log" -) - -// GetMachinesByMachineDeploymentsInput is the input for GetMachinesByMachineDeployments. -type GetMachinesByMachineDeploymentsInput struct { - Lister Lister - ClusterName string - Namespace string - MachineDeployment clusterv1.MachineDeployment -} - -// GetMachinesByMachineDeployments returns Machine objects for a cluster belonging to a machine deployment. -// Important! this method relies on labels that are created by the CAPI controllers during the first reconciliation, so -// it is necessary to ensure this is already happened before calling it. -func GetMachinesByMachineDeployments(ctx context.Context, input GetMachinesByMachineDeploymentsInput) []clusterv1.Machine { - Expect(ctx).NotTo(BeNil(), "ctx is required for GetMachinesByMachineDeployments") - Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Lister can't be nil when calling GetMachinesByMachineDeployments") - Expect(input.ClusterName).ToNot(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling GetMachinesByMachineDeployments") - Expect(input.Namespace).ToNot(BeEmpty(), "Invalid argument. input.Namespace can't be empty when calling GetMachinesByMachineDeployments") - Expect(input.MachineDeployment).ToNot(BeNil(), "Invalid argument. input.MachineDeployment can't be nil when calling GetMachinesByMachineDeployments") - - opts := byClusterOptions(input.ClusterName, input.Namespace) - opts = append(opts, machineDeploymentOptions(input.MachineDeployment)...) - - machineList := &clusterv1.MachineList{} - Eventually(func() error { - return input.Lister.List(ctx, machineList, opts...) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineList object for Cluster %s", klog.KRef(input.Namespace, input.ClusterName)) - - return machineList.Items -} - -// GetMachinesByMachineHealthCheckInput is the input for GetMachinesByMachineHealthCheck. -type GetMachinesByMachineHealthCheckInput struct { - Lister Lister - ClusterName string - MachineHealthCheck *clusterv1.MachineHealthCheck -} - -// GetMachinesByMachineHealthCheck returns Machine objects for a cluster that match with MachineHealthCheck selector. -func GetMachinesByMachineHealthCheck(ctx context.Context, input GetMachinesByMachineHealthCheckInput) []clusterv1.Machine { - Expect(ctx).NotTo(BeNil(), "ctx is required for GetMachinesByMachineDeployments") - Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Lister can't be nil when calling GetMachinesByMachineHealthCheck") - Expect(input.ClusterName).ToNot(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling GetMachinesByMachineHealthCheck") - Expect(input.MachineHealthCheck).ToNot(BeNil(), "Invalid argument. input.MachineHealthCheck can't be nil when calling GetMachinesByMachineHealthCheck") - - opts := byClusterOptions(input.ClusterName, input.MachineHealthCheck.Namespace) - opts = append(opts, machineHealthCheckOptions(*input.MachineHealthCheck)...) - - machineList := &clusterv1.MachineList{} - Eventually(func() error { - return input.Lister.List(ctx, machineList, opts...) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineList object for Cluster %s", klog.KRef(input.MachineHealthCheck.Namespace, input.ClusterName)) - - return machineList.Items -} - -// GetControlPlaneMachinesByClusterInput is the input for GetControlPlaneMachinesByCluster. -type GetControlPlaneMachinesByClusterInput struct { - Lister Lister - ClusterName string - Namespace string -} - -// GetControlPlaneMachinesByCluster returns the Machine objects for a cluster. -// Important! this method relies on labels that are created by the CAPI controllers during the first reconciliation, so -// it is necessary to ensure this is already happened before calling it. -func GetControlPlaneMachinesByCluster(ctx context.Context, input GetControlPlaneMachinesByClusterInput) []clusterv1.Machine { - Expect(ctx).NotTo(BeNil(), "ctx is required for GetControlPlaneMachinesByCluster") - Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Lister can't be nil when calling GetControlPlaneMachinesByCluster") - Expect(input.ClusterName).ToNot(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling GetControlPlaneMachinesByCluster") - Expect(input.Namespace).ToNot(BeEmpty(), "Invalid argument. input.Namespace can't be empty when calling GetControlPlaneMachinesByCluster") - - options := append(byClusterOptions(input.ClusterName, input.Namespace), controlPlaneMachineOptions()...) - - machineList := &clusterv1.MachineList{} - Eventually(func() error { - return input.Lister.List(ctx, machineList, options...) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineList object for Cluster %s", klog.KRef(input.Namespace, input.ClusterName)) - - return machineList.Items -} - -// WaitForControlPlaneMachinesToBeUpgradedInput is the input for WaitForControlPlaneMachinesToBeUpgraded. -type WaitForControlPlaneMachinesToBeUpgradedInput struct { - Lister Lister - Cluster *clusterv1.Cluster - KubernetesUpgradeVersion string - MachineCount int -} - -// WaitForControlPlaneMachinesToBeUpgraded waits until all machines are upgraded to the correct Kubernetes version. -func WaitForControlPlaneMachinesToBeUpgraded(ctx context.Context, input WaitForControlPlaneMachinesToBeUpgradedInput, intervals ...interface{}) { - Expect(ctx).NotTo(BeNil(), "ctx is required for WaitForControlPlaneMachinesToBeUpgraded") - Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Lister can't be nil when calling WaitForControlPlaneMachinesToBeUpgraded") - Expect(input.KubernetesUpgradeVersion).ToNot(BeEmpty(), "Invalid argument. input.KubernetesUpgradeVersion can't be empty when calling WaitForControlPlaneMachinesToBeUpgraded") - Expect(input.MachineCount).To(BeNumerically(">", 0), "Invalid argument. input.MachineCount can't be smaller than 1 when calling WaitForControlPlaneMachinesToBeUpgraded") - - Byf("Ensuring all control-plane machines have upgraded kubernetes version %s", input.KubernetesUpgradeVersion) - - Eventually(func() (int, error) { - machines := GetControlPlaneMachinesByCluster(ctx, GetControlPlaneMachinesByClusterInput{ - Lister: input.Lister, - ClusterName: input.Cluster.Name, - Namespace: input.Cluster.Namespace, - }) - - upgraded := 0 - for _, machine := range machines { - m := machine - if *m.Spec.Version == input.KubernetesUpgradeVersion && conditions.IsTrue(&m, clusterv1.MachineNodeHealthyCondition) { - upgraded++ - } - } - if len(machines) > upgraded { - return 0, errors.New("old nodes remain") - } - return upgraded, nil - }, intervals...).Should(Equal(input.MachineCount), "Timed out waiting for all control-plane machines in Cluster %s to be upgraded to kubernetes version %s", klog.KObj(input.Cluster), input.KubernetesUpgradeVersion) -} - -// WaitForMachineDeploymentMachinesToBeUpgradedInput is the input for WaitForMachineDeploymentMachinesToBeUpgraded. -type WaitForMachineDeploymentMachinesToBeUpgradedInput struct { - Lister Lister - Cluster *clusterv1.Cluster - KubernetesUpgradeVersion string - MachineCount int - MachineDeployment clusterv1.MachineDeployment -} - -// WaitForMachineDeploymentMachinesToBeUpgraded waits until all machines belonging to a MachineDeployment are upgraded to the correct kubernetes version. -func WaitForMachineDeploymentMachinesToBeUpgraded(ctx context.Context, input WaitForMachineDeploymentMachinesToBeUpgradedInput, intervals ...interface{}) { - Expect(ctx).NotTo(BeNil(), "ctx is required for WaitForMachineDeploymentMachinesToBeUpgraded") - Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Getter can't be nil when calling WaitForMachineDeploymentMachinesToBeUpgraded") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling WaitForMachineDeploymentMachinesToBeUpgraded") - Expect(input.KubernetesUpgradeVersion).ToNot(BeNil(), "Invalid argument. input.KubernetesUpgradeVersion can't be nil when calling WaitForMachineDeploymentMachinesToBeUpgraded") - Expect(input.MachineDeployment).ToNot(BeNil(), "Invalid argument. input.MachineDeployment can't be nil when calling WaitForMachineDeploymentMachinesToBeUpgraded") - Expect(input.MachineCount).To(BeNumerically(">", 0), "Invalid argument. input.MachineCount can't be smaller than 1 when calling WaitForMachineDeploymentMachinesToBeUpgraded") - - log.Logf("Ensuring all MachineDeployment Machines have upgraded kubernetes version %s", input.KubernetesUpgradeVersion) - Eventually(func() (int, error) { - machines := GetMachinesByMachineDeployments(ctx, GetMachinesByMachineDeploymentsInput{ - Lister: input.Lister, - ClusterName: input.Cluster.Name, - Namespace: input.Cluster.Namespace, - MachineDeployment: input.MachineDeployment, - }) - - upgraded := 0 - for _, machine := range machines { - if *machine.Spec.Version == input.KubernetesUpgradeVersion { - upgraded++ - } - } - if len(machines) > upgraded { - return 0, errors.New("old nodes remain") - } - return upgraded, nil - }, intervals...).Should(Equal(input.MachineCount), "Timed out waiting for all MachineDeployment %s Machines to be upgraded to kubernetes version %s", klog.KObj(&input.MachineDeployment), input.KubernetesUpgradeVersion) -} - -// PatchNodeConditionInput is the input for PatchNodeCondition. -type PatchNodeConditionInput struct { - ClusterProxy ClusterProxy - Cluster *clusterv1.Cluster - NodeCondition corev1.NodeCondition - Machine clusterv1.Machine -} - -// PatchNodeCondition patches a node condition to any one of the machines with a node ref. -func PatchNodeCondition(ctx context.Context, input PatchNodeConditionInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for PatchNodeConditions") - Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling PatchNodeConditions") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling PatchNodeConditions") - Expect(input.NodeCondition).ToNot(BeNil(), "Invalid argument. input.NodeCondition can't be nil when calling PatchNodeConditions") - Expect(input.Machine).ToNot(BeNil(), "Invalid argument. input.Machine can't be nil when calling PatchNodeConditions") - - log.Logf("Patching the node condition to the node") - Expect(input.Machine.Status.NodeRef).ToNot(BeNil()) - node := &corev1.Node{} - Eventually(func() error { - return input.ClusterProxy.GetWorkloadCluster(ctx, input.Cluster.Namespace, input.Cluster.Name).GetClient().Get(ctx, types.NamespacedName{Name: input.Machine.Status.NodeRef.Name, Namespace: input.Machine.Status.NodeRef.Namespace}, node) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get node %s", input.Machine.Status.NodeRef.Name) - patchHelper, err := patch.NewHelper(node, input.ClusterProxy.GetWorkloadCluster(ctx, input.Cluster.Namespace, input.Cluster.Name).GetClient()) - Expect(err).ToNot(HaveOccurred()) - node.Status.Conditions = append(node.Status.Conditions, input.NodeCondition) - Eventually(func() error { - return patchHelper.Patch(ctx, node) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to patch node %s", input.Machine.Status.NodeRef.Name) -} - -// MachineStatusCheck is a type that operates a status check on a Machine. -type MachineStatusCheck func(p *clusterv1.Machine) error - -// WaitForMachineStatusCheckInput is the input for WaitForMachineStatusCheck. -type WaitForMachineStatusCheckInput struct { - Getter Getter - Machine *clusterv1.Machine - StatusChecks []MachineStatusCheck -} - -// WaitForMachineStatusCheck waits for the specified status to be true for the machine. -func WaitForMachineStatusCheck(ctx context.Context, input WaitForMachineStatusCheckInput, intervals ...interface{}) { - Expect(ctx).NotTo(BeNil(), "ctx is required for WaitForMachineStatusCheck") - Expect(input.Machine).ToNot(BeNil(), "Invalid argument. input.Machine can't be nil when calling WaitForMachineStatusCheck") - Expect(input.StatusChecks).ToNot(BeEmpty(), "Invalid argument. input.StatusCheck can't be empty when calling WaitForMachineStatusCheck") - - Eventually(func() (bool, error) { - machine := &clusterv1.Machine{} - key := client.ObjectKey{ - Namespace: input.Machine.Namespace, - Name: input.Machine.Name, - } - err := input.Getter.Get(ctx, key, machine) - Expect(err).NotTo(HaveOccurred()) - - for _, statusCheck := range input.StatusChecks { - err := statusCheck(machine) - if err != nil { - return false, err - } - } - return true, nil - }, intervals...).Should(BeTrue()) -} - -// MachineNodeRefCheck is a MachineStatusCheck ensuring that a NodeRef is assigned to the machine. -func MachineNodeRefCheck() MachineStatusCheck { - return func(machine *clusterv1.Machine) error { - if machine.Status.NodeRef == nil { - return errors.Errorf("NodeRef is not assigned to the machine %s", klog.KObj(machine)) - } - return nil - } -} - -// MachinePhaseCheck is a MachineStatusCheck ensuring that a machines is in the expected phase. -func MachinePhaseCheck(expectedPhase string) MachineStatusCheck { - return func(machine *clusterv1.Machine) error { - if machine.Status.Phase != expectedPhase { - return errors.Errorf("Machine %s is not in phase %s", klog.KObj(machine), expectedPhase) - } - return nil - } -} diff --git a/test/e2e/framework/machinedeployment_helpers.go b/test/e2e/framework/machinedeployment_helpers.go deleted file mode 100644 index 9d842a4a..00000000 --- a/test/e2e/framework/machinedeployment_helpers.go +++ /dev/null @@ -1,498 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "context" - "fmt" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/klog/v2" - "k8s.io/utils/pointer" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" - "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/cluster-api/util/patch" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/kubesphere/kubekey/v3/test/e2e/framework/internal/log" -) - -// CreateMachineDeploymentInput is the input for CreateMachineDeployment. -type CreateMachineDeploymentInput struct { - Creator Creator - MachineDeployment *clusterv1.MachineDeployment - BootstrapConfigTemplate client.Object - InfraMachineTemplate client.Object -} - -// CreateMachineDeployment creates the machine deployment and dependencies. -func CreateMachineDeployment(ctx context.Context, input CreateMachineDeploymentInput) { - By("creating a core MachineDeployment resource") - Eventually(func() error { - return input.Creator.Create(ctx, input.MachineDeployment) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to create MachineDeployment %s", klog.KObj(input.MachineDeployment)) - - By("creating a BootstrapConfigTemplate resource") - Eventually(func() error { - return input.Creator.Create(ctx, input.BootstrapConfigTemplate) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to create BootstrapConfigTemplate %s", klog.KObj(input.BootstrapConfigTemplate)) - - By("creating an InfrastructureMachineTemplate resource") - Eventually(func() error { - return input.Creator.Create(ctx, input.InfraMachineTemplate) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to create InfrastructureMachineTemplate %s", klog.KObj(input.InfraMachineTemplate)) -} - -// GetMachineDeploymentsByClusterInput is the input for GetMachineDeploymentsByCluster. -type GetMachineDeploymentsByClusterInput struct { - Lister Lister - ClusterName string - Namespace string -} - -// GetMachineDeploymentsByCluster returns the MachineDeployments objects for a cluster. -// Important! this method relies on labels that are created by the CAPI controllers during the first reconciliation, so -// it is necessary to ensure this is already happened before calling it. -func GetMachineDeploymentsByCluster(ctx context.Context, input GetMachineDeploymentsByClusterInput) []*clusterv1.MachineDeployment { - deploymentList := &clusterv1.MachineDeploymentList{} - Eventually(func() error { - return input.Lister.List(ctx, deploymentList, byClusterOptions(input.ClusterName, input.Namespace)...) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineDeployments object for Cluster %s", klog.KRef(input.Namespace, input.ClusterName)) - - deployments := make([]*clusterv1.MachineDeployment, len(deploymentList.Items)) - for i := range deploymentList.Items { - Expect(deploymentList.Items[i].Spec.Replicas).ToNot(BeNil()) - deployments[i] = &deploymentList.Items[i] - } - return deployments -} - -// WaitForMachineDeploymentNodesToExistInput is the input for WaitForMachineDeploymentNodesToExist. -type WaitForMachineDeploymentNodesToExistInput struct { - Lister Lister - Cluster *clusterv1.Cluster - MachineDeployment *clusterv1.MachineDeployment -} - -// WaitForMachineDeploymentNodesToExist waits until all nodes associated with a machine deployment exist. -func WaitForMachineDeploymentNodesToExist(ctx context.Context, input WaitForMachineDeploymentNodesToExistInput, intervals ...interface{}) { - Expect(ctx).NotTo(BeNil(), "ctx is required for WaitForMachineDeploymentNodesToExist") - Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Lister can't be nil when calling WaitForMachineDeploymentNodesToExist") - Expect(input.MachineDeployment).ToNot(BeNil(), "Invalid argument. input.MachineDeployment can't be nil when calling WaitForMachineDeploymentNodesToExist") - - By("Waiting for the workload nodes to exist") - Eventually(func() (int, error) { - selectorMap, err := metav1.LabelSelectorAsMap(&input.MachineDeployment.Spec.Selector) - if err != nil { - return 0, err - } - ms := &clusterv1.MachineSetList{} - if err := input.Lister.List(ctx, ms, client.InNamespace(input.Cluster.Namespace), client.MatchingLabels(selectorMap)); err != nil { - return 0, err - } - if len(ms.Items) == 0 { - return 0, errors.New("no machinesets were found") - } - machineSet := ms.Items[0] - selectorMap, err = metav1.LabelSelectorAsMap(&machineSet.Spec.Selector) - if err != nil { - return 0, err - } - machines := &clusterv1.MachineList{} - if err := input.Lister.List(ctx, machines, client.InNamespace(machineSet.Namespace), client.MatchingLabels(selectorMap)); err != nil { - return 0, err - } - count := 0 - for _, machine := range machines.Items { - if machine.Status.NodeRef != nil { - count++ - } - } - return count, nil - }, intervals...).Should(Equal(int(*input.MachineDeployment.Spec.Replicas)), "Timed out waiting for %d nodes to be created for MachineDeployment %s", int(*input.MachineDeployment.Spec.Replicas), klog.KObj(input.MachineDeployment)) -} - -// AssertMachineDeploymentFailureDomainsInput is the input for AssertMachineDeploymentFailureDomains. -type AssertMachineDeploymentFailureDomainsInput struct { - Lister Lister - Cluster *clusterv1.Cluster - MachineDeployment *clusterv1.MachineDeployment -} - -// AssertMachineDeploymentFailureDomains will look at all MachineDeployment machines and see what failure domains they were -// placed in. If machines were placed in unexpected or wrong failure domains the expectation will fail. -func AssertMachineDeploymentFailureDomains(ctx context.Context, input AssertMachineDeploymentFailureDomainsInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for AssertMachineDeploymentFailureDomains") - Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Lister can't be nil when calling AssertMachineDeploymentFailureDomains") - Expect(input.MachineDeployment).ToNot(BeNil(), "Invalid argument. input.MachineDeployment can't be nil when calling AssertMachineDeploymentFailureDomains") - - machineDeploymentFD := pointer.StringDeref(input.MachineDeployment.Spec.Template.Spec.FailureDomain, "") - - Byf("Checking all the machines controlled by %s are in the %q failure domain", input.MachineDeployment.Name, machineDeploymentFD) - selectorMap, err := metav1.LabelSelectorAsMap(&input.MachineDeployment.Spec.Selector) - Expect(err).NotTo(HaveOccurred()) - - ms := &clusterv1.MachineSetList{} - Eventually(func() error { - return input.Lister.List(ctx, ms, client.InNamespace(input.Cluster.Namespace), client.MatchingLabels(selectorMap)) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineSets for Cluster %s", klog.KObj(input.Cluster)) - - for _, machineSet := range ms.Items { - machineSetFD := pointer.StringDeref(machineSet.Spec.Template.Spec.FailureDomain, "") - Expect(machineSetFD).To(Equal(machineDeploymentFD), "MachineSet %s is in the %q failure domain, expecting %q", machineSet.Name, machineSetFD, machineDeploymentFD) - - selectorMap, err = metav1.LabelSelectorAsMap(&machineSet.Spec.Selector) - Expect(err).NotTo(HaveOccurred()) - - machines := &clusterv1.MachineList{} - Eventually(func() error { - return input.Lister.List(ctx, machines, client.InNamespace(machineSet.Namespace), client.MatchingLabels(selectorMap)) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list Machines for Cluster %s", klog.KObj(input.Cluster)) - - for _, machine := range machines.Items { - machineFD := pointer.StringDeref(machine.Spec.FailureDomain, "") - Expect(machineFD).To(Equal(machineDeploymentFD), "Machine %s is in the %q failure domain, expecting %q", machine.Name, machineFD, machineDeploymentFD) - } - } -} - -// DiscoveryAndWaitForMachineDeploymentsInput is the input type for DiscoveryAndWaitForMachineDeployments. -type DiscoveryAndWaitForMachineDeploymentsInput struct { - Lister Lister - Cluster *clusterv1.Cluster -} - -// DiscoveryAndWaitForMachineDeployments discovers the MachineDeployments existing in a cluster and waits for them to be ready (all the machine provisioned). -func DiscoveryAndWaitForMachineDeployments(ctx context.Context, input DiscoveryAndWaitForMachineDeploymentsInput, intervals ...interface{}) []*clusterv1.MachineDeployment { - Expect(ctx).NotTo(BeNil(), "ctx is required for DiscoveryAndWaitForMachineDeployments") - Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Lister can't be nil when calling DiscoveryAndWaitForMachineDeployments") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling DiscoveryAndWaitForMachineDeployments") - - machineDeployments := GetMachineDeploymentsByCluster(ctx, GetMachineDeploymentsByClusterInput{ - Lister: input.Lister, - ClusterName: input.Cluster.Name, - Namespace: input.Cluster.Namespace, - }) - for _, deployment := range machineDeployments { - WaitForMachineDeploymentNodesToExist(ctx, WaitForMachineDeploymentNodesToExistInput{ - Lister: input.Lister, - Cluster: input.Cluster, - MachineDeployment: deployment, - }, intervals...) - - AssertMachineDeploymentFailureDomains(ctx, AssertMachineDeploymentFailureDomainsInput{ - Lister: input.Lister, - Cluster: input.Cluster, - MachineDeployment: deployment, - }) - } - return machineDeployments -} - -// UpgradeMachineDeploymentsAndWaitInput is the input type for UpgradeMachineDeploymentsAndWait. -type UpgradeMachineDeploymentsAndWaitInput struct { - ClusterProxy ClusterProxy - Cluster *clusterv1.Cluster - UpgradeVersion string - UpgradeMachineTemplate *string - MachineDeployments []*clusterv1.MachineDeployment - WaitForMachinesToBeUpgraded []interface{} -} - -// UpgradeMachineDeploymentsAndWait upgrades a machine deployment and waits for its machines to be upgraded. -func UpgradeMachineDeploymentsAndWait(ctx context.Context, input UpgradeMachineDeploymentsAndWaitInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for UpgradeMachineDeploymentsAndWait") - Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling UpgradeMachineDeploymentsAndWait") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling UpgradeMachineDeploymentsAndWait") - Expect(input.UpgradeVersion).ToNot(BeNil(), "Invalid argument. input.UpgradeVersion can't be nil when calling UpgradeMachineDeploymentsAndWait") - Expect(input.MachineDeployments).ToNot(BeEmpty(), "Invalid argument. input.MachineDeployments can't be empty when calling UpgradeMachineDeploymentsAndWait") - - mgmtClient := input.ClusterProxy.GetClient() - - for _, deployment := range input.MachineDeployments { - log.Logf("Patching the new kubernetes version to Machine Deployment %s", klog.KObj(deployment)) - patchHelper, err := patch.NewHelper(deployment, mgmtClient) - Expect(err).ToNot(HaveOccurred()) - - oldVersion := deployment.Spec.Template.Spec.Version - deployment.Spec.Template.Spec.Version = &input.UpgradeVersion - if input.UpgradeMachineTemplate != nil { - deployment.Spec.Template.Spec.InfrastructureRef.Name = *input.UpgradeMachineTemplate - } - Eventually(func() error { - return patchHelper.Patch(ctx, deployment) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to patch Kubernetes version on MachineDeployment %s", klog.KObj(deployment)) - - log.Logf("Waiting for Kubernetes versions of machines in MachineDeployment %s to be upgraded from %s to %s", - klog.KObj(deployment), *oldVersion, input.UpgradeVersion) - WaitForMachineDeploymentMachinesToBeUpgraded(ctx, WaitForMachineDeploymentMachinesToBeUpgradedInput{ - Lister: mgmtClient, - Cluster: input.Cluster, - MachineCount: int(*deployment.Spec.Replicas), - KubernetesUpgradeVersion: input.UpgradeVersion, - MachineDeployment: *deployment, - }, input.WaitForMachinesToBeUpgraded...) - } -} - -// WaitForMachineDeploymentRollingUpgradeToStartInput is the input for WaitForMachineDeploymentRollingUpgradeToStart. -type WaitForMachineDeploymentRollingUpgradeToStartInput struct { - Getter Getter - MachineDeployment *clusterv1.MachineDeployment -} - -// WaitForMachineDeploymentRollingUpgradeToStart waits until rolling upgrade starts. -func WaitForMachineDeploymentRollingUpgradeToStart(ctx context.Context, input WaitForMachineDeploymentRollingUpgradeToStartInput, intervals ...interface{}) { - Expect(ctx).NotTo(BeNil(), "ctx is required for WaitForMachineDeploymentRollingUpgradeToStart") - Expect(input.Getter).ToNot(BeNil(), "Invalid argument. input.Getter can't be nil when calling WaitForMachineDeploymentRollingUpgradeToStart") - Expect(input.MachineDeployment).ToNot(BeNil(), "Invalid argument. input.MachineDeployment can't be nil when calling WaitForMachineDeploymentRollingUpgradeToStarts") - - log.Logf("Waiting for MachineDeployment rolling upgrade to start") - Eventually(func(g Gomega) bool { - md := &clusterv1.MachineDeployment{} - g.Expect(input.Getter.Get(ctx, client.ObjectKey{Namespace: input.MachineDeployment.Namespace, Name: input.MachineDeployment.Name}, md)).To(Succeed()) - return md.Status.Replicas != md.Status.AvailableReplicas - }, intervals...).Should(BeTrue()) -} - -// WaitForMachineDeploymentRollingUpgradeToCompleteInput is the input for WaitForMachineDeploymentRollingUpgradeToComplete. -type WaitForMachineDeploymentRollingUpgradeToCompleteInput struct { - Getter Getter - MachineDeployment *clusterv1.MachineDeployment -} - -// WaitForMachineDeploymentRollingUpgradeToComplete waits until rolling upgrade is complete. -func WaitForMachineDeploymentRollingUpgradeToComplete(ctx context.Context, input WaitForMachineDeploymentRollingUpgradeToCompleteInput, intervals ...interface{}) { - Expect(ctx).NotTo(BeNil(), "ctx is required for WaitForMachineDeploymentRollingUpgradeToComplete") - Expect(input.Getter).ToNot(BeNil(), "Invalid argument. input.Getter can't be nil when calling WaitForMachineDeploymentRollingUpgradeToComplete") - Expect(input.MachineDeployment).ToNot(BeNil(), "Invalid argument. input.MachineDeployment can't be nil when calling WaitForMachineDeploymentRollingUpgradeToComplete") - - log.Logf("Waiting for MachineDeployment rolling upgrade to complete") - Eventually(func(g Gomega) bool { - md := &clusterv1.MachineDeployment{} - g.Expect(input.Getter.Get(ctx, client.ObjectKey{Namespace: input.MachineDeployment.Namespace, Name: input.MachineDeployment.Name}, md)).To(Succeed()) - return md.Status.Replicas == md.Status.AvailableReplicas - }, intervals...).Should(BeTrue()) -} - -// UpgradeMachineDeploymentInfrastructureRefAndWaitInput is the input type for UpgradeMachineDeploymentInfrastructureRefAndWait. -type UpgradeMachineDeploymentInfrastructureRefAndWaitInput struct { - ClusterProxy ClusterProxy - Cluster *clusterv1.Cluster - MachineDeployments []*clusterv1.MachineDeployment - WaitForMachinesToBeUpgraded []interface{} -} - -// UpgradeMachineDeploymentInfrastructureRefAndWait upgrades a machine deployment infrastructure ref and waits for its machines to be upgraded. -func UpgradeMachineDeploymentInfrastructureRefAndWait(ctx context.Context, input UpgradeMachineDeploymentInfrastructureRefAndWaitInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for UpgradeMachineDeploymentInfrastructureRefAndWait") - Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling UpgradeMachineDeploymentInfrastructureRefAndWait") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling UpgradeMachineDeploymentInfrastructureRefAndWait") - Expect(input.MachineDeployments).ToNot(BeEmpty(), "Invalid argument. input.MachineDeployments can't be empty when calling UpgradeMachineDeploymentInfrastructureRefAndWait") - - mgmtClient := input.ClusterProxy.GetClient() - - for _, deployment := range input.MachineDeployments { - log.Logf("Patching the new infrastructure ref to Machine Deployment %s", klog.KObj(deployment)) - // Retrieve infra object - infraRef := deployment.Spec.Template.Spec.InfrastructureRef - infraObj := &unstructured.Unstructured{} - infraObj.SetGroupVersionKind(infraRef.GroupVersionKind()) - key := client.ObjectKey{ - Namespace: input.Cluster.Namespace, - Name: infraRef.Name, - } - Eventually(func() error { - return mgmtClient.Get(ctx, key, infraObj) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get infra object %s for MachineDeployment %s", klog.KRef(key.Namespace, key.Name), klog.KObj(deployment)) - - // Creates a new infra object - newInfraObj := infraObj - newInfraObjName := fmt.Sprintf("%s-%s", infraRef.Name, util.RandomString(6)) - newInfraObj.SetName(newInfraObjName) - newInfraObj.SetResourceVersion("") - Eventually(func() error { - return mgmtClient.Create(ctx, newInfraObj) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to create new infrastructure object %s for MachineDeployment %s", klog.KObj(infraObj), klog.KObj(deployment)) - - // Patch the new infra object's ref to the machine deployment - patchHelper, err := patch.NewHelper(deployment, mgmtClient) - Expect(err).ToNot(HaveOccurred()) - infraRef.Name = newInfraObjName - deployment.Spec.Template.Spec.InfrastructureRef = infraRef - Eventually(func() error { - return patchHelper.Patch(ctx, deployment) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to patch new infrastructure ref to MachineDeployment %s", klog.KObj(deployment)) - - log.Logf("Waiting for rolling upgrade to start.") - WaitForMachineDeploymentRollingUpgradeToStart(ctx, WaitForMachineDeploymentRollingUpgradeToStartInput{ - Getter: mgmtClient, - MachineDeployment: deployment, - }, input.WaitForMachinesToBeUpgraded...) - - log.Logf("Waiting for rolling upgrade to complete.") - WaitForMachineDeploymentRollingUpgradeToComplete(ctx, WaitForMachineDeploymentRollingUpgradeToCompleteInput{ - Getter: mgmtClient, - MachineDeployment: deployment, - }, input.WaitForMachinesToBeUpgraded...) - } -} - -// machineDeploymentOptions returns a set of ListOptions that allows to get all machine objects belonging to a machine deployment. -func machineDeploymentOptions(deployment clusterv1.MachineDeployment) []client.ListOption { - return []client.ListOption{ - client.MatchingLabels(deployment.Spec.Selector.MatchLabels), - } -} - -// ScaleAndWaitMachineDeploymentInput is the input for ScaleAndWaitMachineDeployment. -type ScaleAndWaitMachineDeploymentInput struct { - ClusterProxy ClusterProxy - Cluster *clusterv1.Cluster - MachineDeployment *clusterv1.MachineDeployment - Replicas int32 - WaitForMachineDeployments []interface{} -} - -// ScaleAndWaitMachineDeployment scales MachineDeployment and waits until all machines have node ref and equal to Replicas. -func ScaleAndWaitMachineDeployment(ctx context.Context, input ScaleAndWaitMachineDeploymentInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for ScaleAndWaitMachineDeployment") - Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling ScaleAndWaitMachineDeployment") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling ScaleAndWaitMachineDeployment") - - log.Logf("Scaling machine deployment %s from %d to %d replicas", klog.KObj(input.MachineDeployment), *input.MachineDeployment.Spec.Replicas, input.Replicas) - patchHelper, err := patch.NewHelper(input.MachineDeployment, input.ClusterProxy.GetClient()) - Expect(err).ToNot(HaveOccurred()) - input.MachineDeployment.Spec.Replicas = pointer.Int32(input.Replicas) - Eventually(func() error { - return patchHelper.Patch(ctx, input.MachineDeployment) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to scale machine deployment %s", klog.KObj(input.MachineDeployment)) - - log.Logf("Waiting for correct number of replicas to exist") - Eventually(func() (int, error) { - selectorMap, err := metav1.LabelSelectorAsMap(&input.MachineDeployment.Spec.Selector) - if err != nil { - return -1, err - } - ms := &clusterv1.MachineSetList{} - if err := input.ClusterProxy.GetClient().List(ctx, ms, client.InNamespace(input.Cluster.Namespace), client.MatchingLabels(selectorMap)); err != nil { - return -1, err - } - if len(ms.Items) == 0 { - return -1, errors.New("no machinesets were found") - } - machineSet := ms.Items[0] - selectorMap, err = metav1.LabelSelectorAsMap(&machineSet.Spec.Selector) - if err != nil { - return -1, err - } - machines := &clusterv1.MachineList{} - if err := input.ClusterProxy.GetClient().List(ctx, machines, client.InNamespace(machineSet.Namespace), client.MatchingLabels(selectorMap)); err != nil { - return -1, err - } - nodeRefCount := 0 - for _, machine := range machines.Items { - if machine.Status.NodeRef != nil { - nodeRefCount++ - } - } - if len(machines.Items) != nodeRefCount { - return -1, errors.New("Machine count does not match existing nodes count") - } - return nodeRefCount, nil - }, input.WaitForMachineDeployments...).Should(Equal(int(*input.MachineDeployment.Spec.Replicas)), "Timed out waiting for Machine Deployment %s to have %d replicas", klog.KObj(input.MachineDeployment), *input.MachineDeployment.Spec.Replicas) -} - -// ScaleAndWaitMachineDeploymentTopologyInput is the input for ScaleAndWaitMachineDeployment. -type ScaleAndWaitMachineDeploymentTopologyInput struct { - ClusterProxy ClusterProxy - Cluster *clusterv1.Cluster - Replicas int32 - WaitForMachineDeployments []interface{} -} - -// ScaleAndWaitMachineDeploymentTopology scales MachineDeployment topology and waits until all machines have node ref and equal to Replicas. -func ScaleAndWaitMachineDeploymentTopology(ctx context.Context, input ScaleAndWaitMachineDeploymentTopologyInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for ScaleAndWaitMachineDeployment") - Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling ScaleAndWaitMachineDeployment") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling ScaleAndWaitMachineDeployment") - Expect(input.Cluster.Spec.Topology.Workers).ToNot(BeNil(), "Invalid argument. input.Cluster must have MachineDeployment topologies") - Expect(len(input.Cluster.Spec.Topology.Workers.MachineDeployments) >= 1).To(BeTrue(), "Invalid argument. input.Cluster must have at least one MachineDeployment topology") - - mdTopology := input.Cluster.Spec.Topology.Workers.MachineDeployments[0] - log.Logf("Scaling machine deployment topology %s from %d to %d replicas", mdTopology.Name, *mdTopology.Replicas, input.Replicas) - patchHelper, err := patch.NewHelper(input.Cluster, input.ClusterProxy.GetClient()) - Expect(err).ToNot(HaveOccurred()) - mdTopology.Replicas = pointer.Int32(input.Replicas) - input.Cluster.Spec.Topology.Workers.MachineDeployments[0] = mdTopology - Eventually(func() error { - return patchHelper.Patch(ctx, input.Cluster) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to scale machine deployment topology %s", mdTopology.Name) - - log.Logf("Waiting for correct number of replicas to exist") - deploymentList := &clusterv1.MachineDeploymentList{} - Eventually(func() error { - return input.ClusterProxy.GetClient().List(ctx, deploymentList, - client.InNamespace(input.Cluster.Namespace), - client.MatchingLabels{ - clusterv1.ClusterLabelName: input.Cluster.Name, - clusterv1.ClusterTopologyMachineDeploymentLabelName: mdTopology.Name, - }, - ) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineDeployments object for Cluster %s", klog.KRef(input.Cluster.Namespace, input.Cluster.Name)) - - Expect(deploymentList.Items).To(HaveLen(1)) - md := deploymentList.Items[0] - - Eventually(func() (int, error) { - selectorMap, err := metav1.LabelSelectorAsMap(&md.Spec.Selector) - if err != nil { - return -1, err - } - ms := &clusterv1.MachineSetList{} - if err := input.ClusterProxy.GetClient().List(ctx, ms, client.InNamespace(input.Cluster.Namespace), client.MatchingLabels(selectorMap)); err != nil { - return -1, err - } - if len(ms.Items) == 0 { - return -1, errors.New("no machinesets were found") - } - machineSet := ms.Items[0] - selectorMap, err = metav1.LabelSelectorAsMap(&machineSet.Spec.Selector) - if err != nil { - return -1, err - } - machines := &clusterv1.MachineList{} - if err := input.ClusterProxy.GetClient().List(ctx, machines, client.InNamespace(machineSet.Namespace), client.MatchingLabels(selectorMap)); err != nil { - return -1, err - } - nodeRefCount := 0 - for _, machine := range machines.Items { - if machine.Status.NodeRef != nil { - nodeRefCount++ - } - } - if len(machines.Items) != nodeRefCount { - return -1, errors.New("Machine count does not match existing nodes count") - } - return nodeRefCount, nil - }, input.WaitForMachineDeployments...).Should(Equal(int(*md.Spec.Replicas)), "Timed out waiting for Machine Deployment %s to have %d replicas", klog.KObj(&md), *md.Spec.Replicas) -} diff --git a/test/e2e/framework/machinehealthcheck_helpers.go b/test/e2e/framework/machinehealthcheck_helpers.go deleted file mode 100644 index de2c583b..00000000 --- a/test/e2e/framework/machinehealthcheck_helpers.go +++ /dev/null @@ -1,180 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "context" - "fmt" - "time" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/klog/v2" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// DiscoverMachineHealthCheckAndWaitForRemediationInput is the input for DiscoverMachineHealthCheckAndWait. -type DiscoverMachineHealthCheckAndWaitForRemediationInput struct { - ClusterProxy ClusterProxy - Cluster *clusterv1.Cluster - WaitForMachineRemediation []interface{} -} - -// DiscoverMachineHealthChecksAndWaitForRemediation patches an unhealthy node condition to one node observed by the Machine Health Check and then wait for remediation. -func DiscoverMachineHealthChecksAndWaitForRemediation(ctx context.Context, input DiscoverMachineHealthCheckAndWaitForRemediationInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for DiscoverMachineHealthChecksAndWaitForRemediation") - Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling DiscoverMachineHealthChecksAndWaitForRemediation") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling DiscoverMachineHealthChecksAndWaitForRemediation") - - mgmtClient := input.ClusterProxy.GetClient() - fmt.Fprintln(GinkgoWriter, "Discovering machine health check resources") - machineHealthChecks := GetMachineHealthChecksForCluster(ctx, GetMachineHealthChecksForClusterInput{ - Lister: mgmtClient, - ClusterName: input.Cluster.Name, - Namespace: input.Cluster.Namespace, - }) - - Expect(machineHealthChecks).NotTo(BeEmpty()) - - for _, mhc := range machineHealthChecks { - Expect(mhc.Spec.UnhealthyConditions).NotTo(BeEmpty()) - - fmt.Fprintln(GinkgoWriter, "Ensuring there is at least 1 Machine that MachineHealthCheck is matching") - machines := GetMachinesByMachineHealthCheck(ctx, GetMachinesByMachineHealthCheckInput{ - Lister: mgmtClient, - ClusterName: input.Cluster.Name, - MachineHealthCheck: mhc, - }) - - Expect(machines).NotTo(BeEmpty()) - - fmt.Fprintln(GinkgoWriter, "Patching MachineHealthCheck unhealthy condition to one of the nodes") - unhealthyNodeCondition := corev1.NodeCondition{ - Type: mhc.Spec.UnhealthyConditions[0].Type, - Status: mhc.Spec.UnhealthyConditions[0].Status, - LastTransitionTime: metav1.Time{Time: time.Now()}, - } - PatchNodeCondition(ctx, PatchNodeConditionInput{ - ClusterProxy: input.ClusterProxy, - Cluster: input.Cluster, - NodeCondition: unhealthyNodeCondition, - Machine: machines[0], - }) - - fmt.Fprintln(GinkgoWriter, "Waiting for remediation") - WaitForMachineHealthCheckToRemediateUnhealthyNodeCondition(ctx, WaitForMachineHealthCheckToRemediateUnhealthyNodeConditionInput{ - ClusterProxy: input.ClusterProxy, - Cluster: input.Cluster, - MachineHealthCheck: mhc, - MachinesCount: len(machines), - }, input.WaitForMachineRemediation...) - } -} - -// GetMachineHealthChecksForClusterInput is the input for GetMachineHealthChecksForCluster. -type GetMachineHealthChecksForClusterInput struct { - Lister Lister - ClusterName string - Namespace string -} - -// GetMachineHealthChecksForCluster returns the MachineHealthCheck objects for a cluster. -// Important! this method relies on labels that are created by the CAPI controllers during the first reconciliation, so -// it is necessary to ensure this is already happened before calling it. -func GetMachineHealthChecksForCluster(ctx context.Context, input GetMachineHealthChecksForClusterInput) []*clusterv1.MachineHealthCheck { - machineHealthCheckList := &clusterv1.MachineHealthCheckList{} - Eventually(func() error { - return input.Lister.List(ctx, machineHealthCheckList, byClusterOptions(input.ClusterName, input.Namespace)...) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineDeployments object for Cluster %s", klog.KRef(input.Namespace, input.ClusterName)) - - machineHealthChecks := make([]*clusterv1.MachineHealthCheck, len(machineHealthCheckList.Items)) - for i := range machineHealthCheckList.Items { - machineHealthChecks[i] = &machineHealthCheckList.Items[i] - } - return machineHealthChecks -} - -// machineHealthCheckOptions returns a set of ListOptions that allows to get all machine objects belonging to a MachineHealthCheck. -func machineHealthCheckOptions(machineHealthCheck clusterv1.MachineHealthCheck) []client.ListOption { - return []client.ListOption{ - client.MatchingLabels(machineHealthCheck.Spec.Selector.MatchLabels), - } -} - -// WaitForMachineHealthCheckToRemediateUnhealthyNodeConditionInput is the input for WaitForMachineHealthCheckToRemediateUnhealthyNodeCondition. -type WaitForMachineHealthCheckToRemediateUnhealthyNodeConditionInput struct { - ClusterProxy ClusterProxy - Cluster *clusterv1.Cluster - MachineHealthCheck *clusterv1.MachineHealthCheck - MachinesCount int -} - -// WaitForMachineHealthCheckToRemediateUnhealthyNodeCondition patches a node condition to any one of the machines with a node ref. -func WaitForMachineHealthCheckToRemediateUnhealthyNodeCondition(ctx context.Context, input WaitForMachineHealthCheckToRemediateUnhealthyNodeConditionInput, intervals ...interface{}) { - Expect(ctx).NotTo(BeNil(), "ctx is required for WaitForMachineHealthCheckToRemediateUnhealthyNodeCondition") - Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling WaitForMachineHealthCheckToRemediateUnhealthyNodeCondition") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling WaitForMachineHealthCheckToRemediateUnhealthyNodeCondition") - Expect(input.MachineHealthCheck).NotTo(BeNil(), "Invalid argument. input.MachineHealthCheck can't be nil when calling WaitForMachineHealthCheckToRemediateUnhealthyNodeCondition") - Expect(input.MachinesCount).NotTo(BeZero(), "Invalid argument. input.MachinesCount can't be zero when calling WaitForMachineHealthCheckToRemediateUnhealthyNodeCondition") - - fmt.Fprintln(GinkgoWriter, "Waiting until the node with unhealthy node condition is remediated") - Eventually(func() bool { - machines := GetMachinesByMachineHealthCheck(ctx, GetMachinesByMachineHealthCheckInput{ - Lister: input.ClusterProxy.GetClient(), - ClusterName: input.Cluster.Name, - MachineHealthCheck: input.MachineHealthCheck, - }) - // Wait for all the machines to exists. - // NOTE: this is required given that this helper is called after a remediation - // and we want to make sure all the machine are back in place before testing for unhealthyCondition being fixed. - if len(machines) < input.MachinesCount { - return false - } - - for _, machine := range machines { - if machine.Status.NodeRef == nil { - return false - } - node := &corev1.Node{} - // This should not be an Expect(), because it may return error during machine deletion. - err := input.ClusterProxy.GetWorkloadCluster(ctx, input.Cluster.Namespace, input.Cluster.Name).GetClient().Get(ctx, types.NamespacedName{Name: machine.Status.NodeRef.Name, Namespace: machine.Status.NodeRef.Namespace}, node) - if err != nil { - return false - } - if hasMatchingUnhealthyConditions(input.MachineHealthCheck, node.Status.Conditions) { - return false - } - } - return true - }, intervals...).Should(BeTrue()) -} - -// hasMatchingUnhealthyConditions returns true if any node condition matches with machine health check unhealthy conditions. -func hasMatchingUnhealthyConditions(machineHealthCheck *clusterv1.MachineHealthCheck, nodeConditions []corev1.NodeCondition) bool { - for _, unhealthyCondition := range machineHealthCheck.Spec.UnhealthyConditions { - for _, nodeCondition := range nodeConditions { - if nodeCondition.Type == unhealthyCondition.Type && nodeCondition.Status == unhealthyCondition.Status { - return true - } - } - } - return false -} diff --git a/test/e2e/framework/machinepool_helpers.go b/test/e2e/framework/machinepool_helpers.go deleted file mode 100644 index 6b635e61..00000000 --- a/test/e2e/framework/machinepool_helpers.go +++ /dev/null @@ -1,306 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "context" - "fmt" - "strings" - - "github.com/blang/semver" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog/v2" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" - "sigs.k8s.io/cluster-api/util/patch" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/kubesphere/kubekey/v3/test/e2e/framework/internal/log" -) - -// GetMachinePoolsByClusterInput is the input for GetMachinePoolsByCluster. -type GetMachinePoolsByClusterInput struct { - Lister Lister - ClusterName string - Namespace string -} - -// GetMachinePoolsByCluster returns the MachinePools objects for a cluster. -// Important! this method relies on labels that are created by the CAPI controllers during the first reconciliation, so -// it is necessary to ensure this is already happened before calling it. -func GetMachinePoolsByCluster(ctx context.Context, input GetMachinePoolsByClusterInput) []*expv1.MachinePool { - Expect(ctx).NotTo(BeNil(), "ctx is required for GetMachinePoolsByCluster") - Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Lister can't be nil when calling GetMachinePoolsByCluster") - Expect(input.Namespace).ToNot(BeEmpty(), "Invalid argument. input.Namespace can't be empty when calling GetMachinePoolsByCluster") - Expect(input.ClusterName).ToNot(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling GetMachinePoolsByCluster") - - mpList := &expv1.MachinePoolList{} - Eventually(func() error { - return input.Lister.List(ctx, mpList, byClusterOptions(input.ClusterName, input.Namespace)...) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachinePools object for Cluster %s", klog.KRef(input.Namespace, input.ClusterName)) - - mps := make([]*expv1.MachinePool, len(mpList.Items)) - for i := range mpList.Items { - mps[i] = &mpList.Items[i] - } - return mps -} - -// WaitForMachinePoolNodesToExistInput is the input for WaitForMachinePoolNodesToExist. -type WaitForMachinePoolNodesToExistInput struct { - Getter Getter - MachinePool *expv1.MachinePool -} - -// WaitForMachinePoolNodesToExist waits until all nodes associated with a machine pool exist. -func WaitForMachinePoolNodesToExist(ctx context.Context, input WaitForMachinePoolNodesToExistInput, intervals ...interface{}) { - Expect(ctx).NotTo(BeNil(), "ctx is required for WaitForMachinePoolNodesToExist") - Expect(input.Getter).ToNot(BeNil(), "Invalid argument. input.Getter can't be nil when calling WaitForMachinePoolNodesToExist") - Expect(input.MachinePool).ToNot(BeNil(), "Invalid argument. input.MachinePool can't be nil when calling WaitForMachinePoolNodesToExist") - - By("Waiting for the machine pool workload nodes") - Eventually(func() (int, error) { - nn := client.ObjectKey{ - Namespace: input.MachinePool.Namespace, - Name: input.MachinePool.Name, - } - - if err := input.Getter.Get(ctx, nn, input.MachinePool); err != nil { - return 0, err - } - - return int(input.MachinePool.Status.ReadyReplicas), nil - }, intervals...).Should(Equal(int(*input.MachinePool.Spec.Replicas)), "Timed out waiting for %v ready replicas for MachinePool %s", *input.MachinePool.Spec.Replicas, klog.KObj(input.MachinePool)) -} - -// DiscoveryAndWaitForMachinePoolsInput is the input type for DiscoveryAndWaitForMachinePools. -type DiscoveryAndWaitForMachinePoolsInput struct { - Getter Getter - Lister Lister - Cluster *clusterv1.Cluster -} - -// DiscoveryAndWaitForMachinePools discovers the MachinePools existing in a cluster and waits for them to be ready (all the machines provisioned). -func DiscoveryAndWaitForMachinePools(ctx context.Context, input DiscoveryAndWaitForMachinePoolsInput, intervals ...interface{}) []*expv1.MachinePool { - Expect(ctx).NotTo(BeNil(), "ctx is required for DiscoveryAndWaitForMachinePools") - Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Lister can't be nil when calling DiscoveryAndWaitForMachinePools") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling DiscoveryAndWaitForMachinePools") - - machinePools := GetMachinePoolsByCluster(ctx, GetMachinePoolsByClusterInput{ - Lister: input.Lister, - ClusterName: input.Cluster.Name, - Namespace: input.Cluster.Namespace, - }) - for _, machinepool := range machinePools { - WaitForMachinePoolNodesToExist(ctx, WaitForMachinePoolNodesToExistInput{ - Getter: input.Getter, - MachinePool: machinepool, - }, intervals...) - - // TODO: check for failure domains; currently MP doesn't provide a way to check where Machine are placed - // (checking infrastructure is the only alternative, but this makes test not portable) - } - return machinePools -} - -// UpgradeMachinePoolAndWaitInput is the input type for UpgradeMachinePoolAndWait. -type UpgradeMachinePoolAndWaitInput struct { - ClusterProxy ClusterProxy - Cluster *clusterv1.Cluster - UpgradeVersion string - MachinePools []*expv1.MachinePool - WaitForMachinePoolToBeUpgraded []interface{} -} - -// UpgradeMachinePoolAndWait upgrades a machine pool and waits for its instances to be upgraded. -func UpgradeMachinePoolAndWait(ctx context.Context, input UpgradeMachinePoolAndWaitInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for UpgradeMachinePoolAndWait") - Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling UpgradeMachinePoolAndWait") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling UpgradeMachinePoolAndWait") - Expect(input.UpgradeVersion).ToNot(BeNil(), "Invalid argument. input.UpgradeVersion can't be nil when calling UpgradeMachinePoolAndWait") - Expect(input.MachinePools).ToNot(BeNil(), "Invalid argument. input.MachinePools can't be empty when calling UpgradeMachinePoolAndWait") - - mgmtClient := input.ClusterProxy.GetClient() - for i := range input.MachinePools { - mp := input.MachinePools[i] - log.Logf("Patching the new Kubernetes version to Machine Pool %s", klog.KObj(mp)) - patchHelper, err := patch.NewHelper(mp, mgmtClient) - Expect(err).ToNot(HaveOccurred()) - - // Store old version. - oldVersion := mp.Spec.Template.Spec.Version - - // Upgrade to new Version. - mp.Spec.Template.Spec.Version = &input.UpgradeVersion - - // Drop "-cgroupfs" suffix from BootstrapConfig ref name, i.e. we switch from a - // BootstrapConfig with pinned cgroupfs cgroupDriver to the regular BootstrapConfig. - // This is a workaround for CAPD, because kind and CAPD only support: - // * cgroupDriver cgroupfs for Kubernetes < v1.24 - // * cgroupDriver systemd for Kubernetes >= v1.24. - // We can remove this as soon as we don't test upgrades from Kubernetes < v1.24 anymore with CAPD - // or MachinePools are supported in ClusterClass. - if mp.Spec.Template.Spec.InfrastructureRef.Kind == "DockerMachinePool" { - version, err := semver.ParseTolerant(input.UpgradeVersion) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to parse UpgradeVersion %q", input.UpgradeVersion)) - if version.GTE(semver.MustParse("1.24.0")) && strings.HasSuffix(mp.Spec.Template.Spec.Bootstrap.ConfigRef.Name, "-cgroupfs") { - mp.Spec.Template.Spec.Bootstrap.ConfigRef.Name = strings.TrimSuffix(mp.Spec.Template.Spec.Bootstrap.ConfigRef.Name, "-cgroupfs") - // We have to set DataSecretName to nil, so the secret of the new bootstrap ConfigRef gets picked up. - mp.Spec.Template.Spec.Bootstrap.DataSecretName = nil - } - } - - Eventually(func() error { - return patchHelper.Patch(ctx, mp) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to patch the new Kubernetes version to Machine Pool %s", klog.KObj(mp)) - - log.Logf("Waiting for Kubernetes versions of machines in MachinePool %s to be upgraded from %s to %s", - klog.KObj(mp), *oldVersion, input.UpgradeVersion) - WaitForMachinePoolInstancesToBeUpgraded(ctx, WaitForMachinePoolInstancesToBeUpgradedInput{ - Getter: mgmtClient, - WorkloadClusterGetter: input.ClusterProxy.GetWorkloadCluster(ctx, input.Cluster.Namespace, input.Cluster.Name).GetClient(), - Cluster: input.Cluster, - MachineCount: int(*mp.Spec.Replicas), - KubernetesUpgradeVersion: input.UpgradeVersion, - MachinePool: mp, - }, input.WaitForMachinePoolToBeUpgraded...) - } -} - -// ScaleMachinePoolAndWaitInput is the input type for ScaleMachinePoolAndWait. -type ScaleMachinePoolAndWaitInput struct { - ClusterProxy ClusterProxy - Cluster *clusterv1.Cluster - Replicas int32 - MachinePools []*expv1.MachinePool - WaitForMachinePoolToScale []interface{} -} - -// ScaleMachinePoolAndWait scales a machine pool and waits for its instances to scale up. -func ScaleMachinePoolAndWait(ctx context.Context, input ScaleMachinePoolAndWaitInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for UpgradeMachinePoolAndWait") - Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling UpgradeMachinePoolAndWait") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling UpgradeMachinePoolAndWait") - Expect(input.MachinePools).ToNot(BeNil(), "Invalid argument. input.MachinePools can't be empty when calling UpgradeMachinePoolAndWait") - - mgmtClient := input.ClusterProxy.GetClient() - for _, mp := range input.MachinePools { - log.Logf("Patching the replica count in Machine Pool %s", klog.KObj(mp)) - patchHelper, err := patch.NewHelper(mp, mgmtClient) - Expect(err).ToNot(HaveOccurred()) - - mp.Spec.Replicas = &input.Replicas - Eventually(func() error { - return patchHelper.Patch(ctx, mp) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to patch MachinePool %s", klog.KObj(mp)) - } - - for _, mp := range input.MachinePools { - WaitForMachinePoolNodesToExist(ctx, WaitForMachinePoolNodesToExistInput{ - Getter: mgmtClient, - MachinePool: mp, - }, input.WaitForMachinePoolToScale...) - } -} - -// WaitForMachinePoolInstancesToBeUpgradedInput is the input for WaitForMachinePoolInstancesToBeUpgraded. -type WaitForMachinePoolInstancesToBeUpgradedInput struct { - Getter Getter - WorkloadClusterGetter Getter - Cluster *clusterv1.Cluster - KubernetesUpgradeVersion string - MachineCount int - MachinePool *expv1.MachinePool -} - -// WaitForMachinePoolInstancesToBeUpgraded waits until all instances belonging to a MachinePool are upgraded to the correct kubernetes version. -func WaitForMachinePoolInstancesToBeUpgraded(ctx context.Context, input WaitForMachinePoolInstancesToBeUpgradedInput, intervals ...interface{}) { - Expect(ctx).NotTo(BeNil(), "ctx is required for WaitForMachinePoolInstancesToBeUpgraded") - Expect(input.Getter).ToNot(BeNil(), "Invalid argument. input.Getter can't be nil when calling WaitForMachinePoolInstancesToBeUpgraded") - Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling WaitForMachinePoolInstancesToBeUpgraded") - Expect(input.KubernetesUpgradeVersion).ToNot(BeNil(), "Invalid argument. input.KubernetesUpgradeVersion can't be nil when calling WaitForMachinePoolInstancesToBeUpgraded") - Expect(input.MachinePool).ToNot(BeNil(), "Invalid argument. input.MachinePool can't be nil when calling WaitForMachinePoolInstancesToBeUpgraded") - Expect(input.MachineCount).To(BeNumerically(">", 0), "Invalid argument. input.MachineCount can't be smaller than 1 when calling WaitForMachinePoolInstancesToBeUpgraded") - - log.Logf("Ensuring all MachinePool Instances have upgraded kubernetes version %s", input.KubernetesUpgradeVersion) - Eventually(func() (int, error) { - nn := client.ObjectKey{ - Namespace: input.MachinePool.Namespace, - Name: input.MachinePool.Name, - } - if err := input.Getter.Get(ctx, nn, input.MachinePool); err != nil { - return 0, err - } - versions := getMachinePoolInstanceVersions(ctx, GetMachinesPoolInstancesInput{ - WorkloadClusterGetter: input.WorkloadClusterGetter, - Namespace: input.Cluster.Namespace, - MachinePool: input.MachinePool, - }) - - matches := 0 - for _, version := range versions { - if version == input.KubernetesUpgradeVersion { - matches++ - } - } - - if matches != len(versions) { - return 0, errors.New("old version instances remain") - } - - return matches, nil - }, intervals...).Should(Equal(input.MachineCount), "Timed out waiting for all MachinePool %s instances to be upgraded to Kubernetes version %s", klog.KObj(input.MachinePool), input.KubernetesUpgradeVersion) -} - -// GetMachinesPoolInstancesInput is the input for GetMachinesPoolInstances. -type GetMachinesPoolInstancesInput struct { - WorkloadClusterGetter Getter - Namespace string - MachinePool *expv1.MachinePool -} - -// getMachinePoolInstanceVersions returns the Kubernetes versions of the machine pool instances. -func getMachinePoolInstanceVersions(ctx context.Context, input GetMachinesPoolInstancesInput) []string { - Expect(ctx).NotTo(BeNil(), "ctx is required for getMachinePoolInstanceVersions") - Expect(input.WorkloadClusterGetter).ToNot(BeNil(), "Invalid argument. input.WorkloadClusterGetter can't be nil when calling getMachinePoolInstanceVersions") - Expect(input.Namespace).ToNot(BeEmpty(), "Invalid argument. input.Namespace can't be empty when calling getMachinePoolInstanceVersions") - Expect(input.MachinePool).ToNot(BeNil(), "Invalid argument. input.MachinePool can't be nil when calling getMachinePoolInstanceVersions") - - instances := input.MachinePool.Status.NodeRefs - versions := make([]string, len(instances)) - for i, instance := range instances { - node := &corev1.Node{} - err := wait.PollImmediate(retryableOperationInterval, retryableOperationTimeout, func() (bool, error) { - err := input.WorkloadClusterGetter.Get(ctx, client.ObjectKey{Name: instance.Name}, node) - if err != nil { - return false, nil //nolint:nilerr - } - return true, nil - }) - if err != nil { - versions[i] = "unknown" - } else { - versions[i] = node.Status.NodeInfo.KubeletVersion - } - } - - return versions -} diff --git a/test/e2e/framework/namespace_helpers.go b/test/e2e/framework/namespace_helpers.go deleted file mode 100644 index e8e388b4..00000000 --- a/test/e2e/framework/namespace_helpers.go +++ /dev/null @@ -1,198 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "context" - "fmt" - "os" - "path" - "path/filepath" - "time" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - "k8s.io/klog/v2" - "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/kubesphere/kubekey/v3/test/e2e/framework/internal/log" -) - -// CreateNamespaceInput is the input type for CreateNamespace. -type CreateNamespaceInput struct { - Creator Creator - Name string -} - -// CreateNamespace is used to create a namespace object. -// If name is empty, a "test-" + util.RandomString(6) name will be generated. -func CreateNamespace(ctx context.Context, input CreateNamespaceInput, intervals ...interface{}) *corev1.Namespace { - Expect(ctx).NotTo(BeNil(), "ctx is required for DeleteNamespace") - Expect(input.Creator).NotTo(BeNil(), "input.Creator is required for CreateNamespace") - if input.Name == "" { - input.Name = fmt.Sprintf("test-%s", util.RandomString(6)) - } - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: input.Name, - }, - } - log.Logf("Creating namespace %s", input.Name) - Eventually(func() error { - return input.Creator.Create(ctx, ns) - }, intervals...).Should(Succeed(), "Failed to create namespace %s", input.Name) - - return ns -} - -// EnsureNamespace verifies if a namespaces exists. If it doesn't it will -// create the namespace. -func EnsureNamespace(ctx context.Context, mgmt client.Client, namespace string) { - ns := &corev1.Namespace{} - err := mgmt.Get(ctx, client.ObjectKey{Name: namespace}, ns) - if err != nil && apierrors.IsNotFound(err) { - ns = &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - }, - } - Eventually(func() error { - return mgmt.Create(ctx, ns) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to create namespace %q", namespace) - } -} - -// DeleteNamespaceInput is the input type for DeleteNamespace. -type DeleteNamespaceInput struct { - Deleter Deleter - Name string -} - -// DeleteNamespace is used to delete namespace object. -func DeleteNamespace(ctx context.Context, input DeleteNamespaceInput, intervals ...interface{}) { - Expect(ctx).NotTo(BeNil(), "ctx is required for DeleteNamespace") - Expect(input.Deleter).NotTo(BeNil(), "input.Deleter is required for DeleteNamespace") - Expect(input.Name).NotTo(BeEmpty(), "input.Name is required for DeleteNamespace") - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: input.Name, - }, - } - log.Logf("Deleting namespace %s", input.Name) - Eventually(func() error { - return input.Deleter.Delete(ctx, ns) - }, intervals...).Should(Succeed(), "Failed to delete namespace %s", input.Name) -} - -// WatchNamespaceEventsInput is the input type for WatchNamespaceEvents. -type WatchNamespaceEventsInput struct { - ClientSet *kubernetes.Clientset - Name string - LogFolder string -} - -// WatchNamespaceEvents creates a watcher that streams namespace events into a file. -// Example usage: -// -// ctx, cancelWatches := context.WithCancel(context.Background()) -// go func() { -// defer GinkgoRecover() -// framework.WatchNamespaceEvents(ctx, framework.WatchNamespaceEventsInput{ -// ClientSet: clientSet, -// Name: namespace.Name, -// LogFolder: logFolder, -// }) -// }() -// defer cancelWatches() -func WatchNamespaceEvents(ctx context.Context, input WatchNamespaceEventsInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for WatchNamespaceEvents") - Expect(input.ClientSet).NotTo(BeNil(), "input.ClientSet is required for WatchNamespaceEvents") - Expect(input.Name).NotTo(BeEmpty(), "input.Name is required for WatchNamespaceEvents") - - logFile := filepath.Clean(path.Join(input.LogFolder, "resources", input.Name, "events.log")) - Expect(os.MkdirAll(filepath.Dir(logFile), 0750)).To(Succeed()) - - f, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) - Expect(err).NotTo(HaveOccurred()) - defer f.Close() - - informerFactory := informers.NewSharedInformerFactoryWithOptions( - input.ClientSet, - 10*time.Minute, - informers.WithNamespace(input.Name), - ) - eventInformer := informerFactory.Core().V1().Events().Informer() - eventInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - e := obj.(*corev1.Event) - _, _ = f.WriteString(fmt.Sprintf("[New Event] %s\n\tresource: %s/%s/%s\n\treason: %s\n\tmessage: %s\n\tfull: %#v\n", - klog.KObj(e), e.InvolvedObject.APIVersion, e.InvolvedObject.Kind, e.InvolvedObject.Name, e.Reason, e.Message, e)) - }, - UpdateFunc: func(_, obj interface{}) { - e := obj.(*corev1.Event) - _, _ = f.WriteString(fmt.Sprintf("[Updated Event] %s\n\tresource: %s/%s/%s\n\treason: %s\n\tmessage: %s\n\tfull: %#v\n", - klog.KObj(e), e.InvolvedObject.APIVersion, e.InvolvedObject.Kind, e.InvolvedObject.Name, e.Reason, e.Message, e)) - }, - DeleteFunc: func(obj interface{}) {}, - }) - - stopInformer := make(chan struct{}) - defer close(stopInformer) - informerFactory.Start(stopInformer) - <-ctx.Done() - stopInformer <- struct{}{} -} - -// CreateNamespaceAndWatchEventsInput is the input type for CreateNamespaceAndWatchEvents. -type CreateNamespaceAndWatchEventsInput struct { - Creator Creator - ClientSet *kubernetes.Clientset - Name string - LogFolder string -} - -// CreateNamespaceAndWatchEvents creates a namespace and setups a watch for the namespace events. -func CreateNamespaceAndWatchEvents(ctx context.Context, input CreateNamespaceAndWatchEventsInput) (*corev1.Namespace, context.CancelFunc) { - Expect(ctx).NotTo(BeNil(), "ctx is required for CreateNamespaceAndWatchEvents") - Expect(input.Creator).ToNot(BeNil(), "Invalid argument. input.Creator can't be nil when calling CreateNamespaceAndWatchEvents") - Expect(input.ClientSet).ToNot(BeNil(), "Invalid argument. input.ClientSet can't be nil when calling ClientSet") - Expect(input.Name).ToNot(BeEmpty(), "Invalid argument. input.Name can't be empty when calling ClientSet") - Expect(os.MkdirAll(input.LogFolder, 0750)).To(Succeed(), "Invalid argument. input.LogFolder can't be created in CreateNamespaceAndWatchEvents") - - namespace := CreateNamespace(ctx, CreateNamespaceInput{Creator: input.Creator, Name: input.Name}, "40s", "10s") - Expect(namespace).ToNot(BeNil(), "Failed to create namespace %q", input.Name) - - log.Logf("Creating event watcher for namespace %q", input.Name) - watchesCtx, cancelWatches := context.WithCancel(ctx) - go func() { - defer GinkgoRecover() - WatchNamespaceEvents(watchesCtx, WatchNamespaceEventsInput{ - ClientSet: input.ClientSet, - Name: namespace.Name, - LogFolder: input.LogFolder, - }) - }() - return namespace, cancelWatches -} diff --git a/test/e2e/framework/pod_helpers.go b/test/e2e/framework/pod_helpers.go deleted file mode 100644 index f5931fc9..00000000 --- a/test/e2e/framework/pod_helpers.go +++ /dev/null @@ -1,94 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "context" - "strings" - - . "github.com/onsi/gomega" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// PodListCondition is a type that operates a condition on a Pod. -type PodListCondition func(p *corev1.PodList) error - -// WaitForPodListConditionInput is the input args for WaitForPodListCondition. -type WaitForPodListConditionInput struct { - Lister Lister - ListOptions *client.ListOptions - Condition PodListCondition -} - -// WaitForPodListCondition waits for the specified condition to be true for all -// pods returned from the list filter. -func WaitForPodListCondition(ctx context.Context, input WaitForPodListConditionInput, intervals ...interface{}) { - Eventually(func() (bool, error) { - podList := &corev1.PodList{} - if err := input.Lister.List(ctx, podList, input.ListOptions); err != nil { - return false, err - } - - // all pods in the list should satisfy the condition - err := input.Condition(podList) - if err != nil { - return false, err - } - return true, nil - }, intervals...).Should(BeTrue()) -} - -// EtcdImageTagCondition returns a podListCondition that ensures the pod image -// contains the specified image tag. -func EtcdImageTagCondition(expectedTag string, expectedCount int) PodListCondition { - return func(pl *corev1.PodList) error { - countWithCorrectTag := 0 - for _, pod := range pl.Items { - if strings.Contains(pod.Spec.Containers[0].Image, expectedTag) { - countWithCorrectTag++ - } - } - if countWithCorrectTag != expectedCount { - return errors.Errorf("etcdImageTagCondition: expected %d pods to have image tag %q, got %d", expectedCount, expectedTag, countWithCorrectTag) - } - - // This check is to ensure that if there are three controlplane nodes, - // then there are only three etcd pods running. Currently, we create a - // new etcd pod before deleting the previous one. So we can have a - // case where there are three etcd pods with the correct tag and one - // left over that has yet to be deleted. - if len(pl.Items) != expectedCount { - return errors.Errorf("etcdImageTagCondition: expected %d pods, got %d", expectedCount, len(pl.Items)) - } - return nil - } -} - -// PhasePodCondition is a podListCondition ensuring that pods are in the expected -// pod phase. -func PhasePodCondition(expectedPhase corev1.PodPhase) PodListCondition { - return func(pl *corev1.PodList) error { - for _, pod := range pl.Items { - if pod.Status.Phase != expectedPhase { - return errors.Errorf("pod %q is not %s", pod.Name, expectedPhase) - } - } - return nil - } -} diff --git a/test/e2e/suites/capkk/common.go b/test/e2e/suites/capkk/common.go deleted file mode 100644 index 5b4fff25..00000000 --- a/test/e2e/suites/capkk/common.go +++ /dev/null @@ -1,36 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package capkk - -import ( - "fmt" - - . "github.com/onsi/ginkgo" -) - -// Test suite constants for e2e config variables. -const ( - KubernetesVersionManagement = "KUBERNETES_VERSION_MANAGEMENT" - CNIPath = "CNI" - CNIResources = "CNI_RESOURCES" - IPFamily = "IP_FAMILY" -) - -// Byf is a wrapper around By that formats its arguments. -func Byf(format string, a ...interface{}) { - By(fmt.Sprintf(format, a...)) -} diff --git a/test/e2e/suites/capkk/doc.go b/test/e2e/suites/capkk/doc.go deleted file mode 100644 index 451f8c6e..00000000 --- a/test/e2e/suites/capkk/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package capkk implements end to end testing. -package capkk diff --git a/test/e2e/suites/capkk/e2e_suite_test.go b/test/e2e/suites/capkk/e2e_suite_test.go deleted file mode 100644 index c78e06be..00000000 --- a/test/e2e/suites/capkk/e2e_suite_test.go +++ /dev/null @@ -1,296 +0,0 @@ -//go:build e2e -// +build e2e - -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package capkk - -import ( - "flag" - "fmt" - "os" - "path/filepath" - "strings" - "testing" - "time" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/test/framework" - "sigs.k8s.io/cluster-api/test/framework/bootstrap" - "sigs.k8s.io/cluster-api/test/framework/clusterctl" - "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" - ctrl "sigs.k8s.io/controller-runtime" -) - -// Test suite flags. -var ( - // configPath is the path to the e2e config file. - configPath string - - // useExistingCluster instructs the test to use the current cluster instead of creating a new one (default discovery rules apply). - useExistingCluster bool - - // artifactFolder is the folder to store e2e test artifacts. - artifactFolder string - - // clusterctlConfig is the file which tests will use as a clusterctl config. - // If it is not set, a local clusterctl repository (including a clusterctl config) will be created automatically. - clusterctlConfig string - - // alsoLogToFile enables additional logging to the 'ginkgo-log.txt' file in the artifact folder. - // These logs also contain timestamps. - alsoLogToFile bool - - // skipCleanup prevents cleanup of test resources e.g. for debug purposes. - skipCleanup bool -) - -// Test suite global vars. -var ( - ctx = ctrl.SetupSignalHandler() - - // e2eConfig to be used for this test, read from configPath. - e2eConfig *clusterctl.E2EConfig - - // clusterctlConfigPath to be used for this test, created by generating a clusterctl local repository - // with the providers specified in the configPath. - clusterctlConfigPath string - - // bootstrapClusterProvider manages provisioning of the bootstrap cluster to be used for the e2e tests. - // Please note that provisioning will be skipped if e2e.use-existing-cluster is provided. - bootstrapClusterProvider bootstrap.ClusterProvider - - // bootstrapClusterProxy allows to interact with the bootstrap cluster to be used for the e2e tests. - bootstrapClusterProxy framework.ClusterProxy -) - -func init() { - flag.StringVar(&configPath, "e2e.config", "", "path to the e2e config file") - flag.StringVar(&artifactFolder, "e2e.artifacts-folder", "", "folder where e2e test artifact should be stored") - flag.BoolVar(&alsoLogToFile, "e2e.also-log-to-file", true, "if true, ginkgo logs are additionally written to the `ginkgo-log.txt` file in the artifacts folder (including timestamps)") - flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped") - flag.StringVar(&clusterctlConfig, "e2e.clusterctl-config", "", "file which tests will use as a clusterctl config. If it is not set, a local clusterctl repository (including a clusterctl config) will be created automatically.") - flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)") - - SetDefaultEventuallyTimeout(30 * time.Minute) - SetDefaultEventuallyPollingInterval(10 * time.Second) -} - -func TestE2E(t *testing.T) { - g := NewWithT(t) - - // If running in prow, make sure to use the artifacts folder that will be reported in test grid (ignoring the value provided by flag). - if prowArtifactFolder, exists := os.LookupEnv("ARTIFACTS"); exists { - artifactFolder = prowArtifactFolder - } - - // ensure the artifacts folder exists - g.Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid test suite argument. Can't create e2e.artifacts-folder %q", artifactFolder) //nolint:gosec - - RegisterFailHandler(Fail) - - if alsoLogToFile { - w, err := ginkgoextensions.EnableFileLogging(filepath.Join(artifactFolder, "ginkgo-log.txt")) - g.Expect(err).ToNot(HaveOccurred()) - defer w.Close() - } - - RunSpecs(t, "capkk-e2e") -} - -// Using a SynchronizedBeforeSuite for controlling how to create resources shared across ParallelNodes (~ginkgo threads). -// The local clusterctl repository & the bootstrap cluster are created once and shared across all the tests. -var _ = SynchronizedBeforeSuite(func() []byte { - // Before all ParallelNodes. - Expect(configPath).To(BeAnExistingFile(), "Invalid test suite argument. e2e.config should be an existing file.") - - By("Initializing a runtime.Scheme with all the GVK relevant for this test") - scheme := initScheme() - - Byf("Loading the e2e test configuration from %q", configPath) - e2eConfig = loadE2EConfig(configPath) - - if clusterctlConfig == "" { - Byf("Creating a clusterctl local repository into %q", artifactFolder) - clusterctlConfigPath = createClusterctlLocalRepository(e2eConfig, filepath.Join(artifactFolder, "repository")) - } else { - Byf("Using existing clusterctl config %q", clusterctlConfig) - clusterctlConfigPath = clusterctlConfig - } - - By("Setting up the bootstrap cluster") - bootstrapClusterProvider, bootstrapClusterProxy = setupBootstrapCluster(e2eConfig, scheme, useExistingCluster) - - By("Initializing the bootstrap cluster") - initBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder) - - return []byte( - strings.Join([]string{ - artifactFolder, - configPath, - clusterctlConfigPath, - bootstrapClusterProxy.GetKubeconfigPath(), - }, ","), - ) -}, func(data []byte) { - // Before each ParallelNode. - - parts := strings.Split(string(data), ",") - Expect(parts).To(HaveLen(4)) - - artifactFolder = parts[0] - configPath = parts[1] - clusterctlConfigPath = parts[2] - kubeconfigPath := parts[3] - - e2eConfig = loadE2EConfig(configPath) - bootstrapClusterProxy = framework.NewClusterProxy("bootstrap", kubeconfigPath, initScheme()) -}) - -// Using a SynchronizedAfterSuite for controlling how to delete resources shared across ParallelNodes (~ginkgo threads). -// The bootstrap cluster is shared across all the tests, so it should be deleted only after all ParallelNodes completes. -// The local clusterctl repository is preserved like everything else created into the artifact folder. -var _ = SynchronizedAfterSuite(func() { - // After each ParallelNode. -}, func() { - // After all ParallelNodes. - - By("Dumping logs from the bootstrap cluster") - dumpBootstrapClusterLogs(bootstrapClusterProxy) - - By("Tearing down the management cluster") - if !skipCleanup { - tearDown(bootstrapClusterProvider, bootstrapClusterProxy) - } -}) - -func initScheme() *runtime.Scheme { - sc := runtime.NewScheme() - framework.TryAddDefaultSchemes(sc) - return sc -} - -func loadE2EConfig(configPath string) *clusterctl.E2EConfig { - config := clusterctl.LoadE2EConfig(ctx, clusterctl.LoadE2EConfigInput{ConfigPath: configPath}) - Expect(config).ToNot(BeNil(), "Failed to load E2E config from %s", configPath) - - return config -} - -func createClusterctlLocalRepository(config *clusterctl.E2EConfig, repositoryFolder string) string { - createRepositoryInput := clusterctl.CreateRepositoryInput{ - E2EConfig: config, - RepositoryFolder: repositoryFolder, - } - - // Ensuring a CNI file is defined in the config and register a FileTransformation to inject the referenced file in place of the CNI_RESOURCES envSubst variable. - Expect(config.Variables).To(HaveKey(CNIPath), "Missing %s variable in the config", CNIPath) - cniPath := config.GetVariable(CNIPath) - Expect(cniPath).To(BeAnExistingFile(), "The %s variable should resolve to an existing file", CNIPath) - - createRepositoryInput.RegisterClusterResourceSetConfigMapTransformation(cniPath, CNIResources) - - clusterctlConfig := clusterctl.CreateRepository(ctx, createRepositoryInput) - Expect(clusterctlConfig).To(BeAnExistingFile(), "The clusterctl config file does not exists in the local repository %s", repositoryFolder) - return clusterctlConfig -} - -func setupBootstrapCluster(config *clusterctl.E2EConfig, scheme *runtime.Scheme, useExistingCluster bool) (bootstrap.ClusterProvider, framework.ClusterProxy) { - var clusterProvider bootstrap.ClusterProvider - kubeconfigPath := "" - if !useExistingCluster { - By("Creating the bootstrap cluster") - clusterProvider = bootstrap.CreateKindBootstrapClusterAndLoadImages(ctx, bootstrap.CreateKindBootstrapClusterAndLoadImagesInput{ - Name: config.ManagementClusterName, - KubernetesVersion: config.GetVariable(KubernetesVersionManagement), - RequiresDockerSock: config.HasDockerProvider(), - Images: config.Images, - IPFamily: config.GetVariable(IPFamily), - LogFolder: filepath.Join(artifactFolder, "kind"), - }) - Expect(clusterProvider).ToNot(BeNil(), "Failed to create a bootstrap cluster") - - kubeconfigPath = clusterProvider.GetKubeconfigPath() - Expect(kubeconfigPath).To(BeAnExistingFile(), "Failed to get the kubeconfig file for the bootstrap cluster") - } else { - By("Using an existing bootstrap cluster") - } - - clusterProxy := framework.NewClusterProxy("bootstrap", kubeconfigPath, scheme) - Expect(clusterProxy).ToNot(BeNil(), "Failed to get a bootstrap cluster proxy") - - return clusterProvider, clusterProxy -} - -func initBootstrapCluster(bootstrapClusterProxy framework.ClusterProxy, config *clusterctl.E2EConfig, clusterctlConfig, artifactFolder string) { - clusterctl.InitManagementClusterAndWatchControllerLogs(ctx, clusterctl.InitManagementClusterAndWatchControllerLogsInput{ - ClusterProxy: bootstrapClusterProxy, - ClusterctlConfigPath: clusterctlConfig, - InfrastructureProviders: config.InfrastructureProviders(), - LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), - }, config.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) -} - -func dumpBootstrapClusterLogs(bootstrapClusterProxy framework.ClusterProxy) { - if bootstrapClusterProxy == nil { - return - } - - clusterLogCollector := bootstrapClusterProxy.GetLogCollector() - if clusterLogCollector == nil { - return - } - - nodes, err := bootstrapClusterProxy.GetClientSet().CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - if err != nil { - fmt.Printf("Failed to get nodes for the bootstrap cluster: %v\n", err) - return - } - - for i := range nodes.Items { - nodeName := nodes.Items[i].GetName() - err = clusterLogCollector.CollectMachineLog( - ctx, - bootstrapClusterProxy.GetClient(), - // The bootstrap cluster is not expected to be a CAPI cluster, so in order to re-use the logCollector, - // we create a fake machine that wraps the node. - // NOTE: This assumes a naming convention between machines and nodes, which e.g. applies to the bootstrap clusters generated with kind. - // This might not work if you are using an existing bootstrap cluster provided by other means. - &clusterv1.Machine{ - Spec: clusterv1.MachineSpec{ClusterName: nodeName}, - ObjectMeta: metav1.ObjectMeta{Name: nodeName}, - }, - filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName(), "machines", nodeName), - ) - if err != nil { - fmt.Printf("Failed to get logs for the bootstrap cluster node %s: %v\n", nodeName, err) - } - } -} - -func tearDown(bootstrapClusterProvider bootstrap.ClusterProvider, bootstrapClusterProxy framework.ClusterProxy) { - if bootstrapClusterProxy != nil { - bootstrapClusterProxy.Dispose(ctx) - } - if bootstrapClusterProvider != nil { - bootstrapClusterProvider.Dispose(ctx) - } -} diff --git a/test/e2e/suites/capkk/quick_start_test.go b/test/e2e/suites/capkk/quick_start_test.go deleted file mode 100644 index 8f0fcc82..00000000 --- a/test/e2e/suites/capkk/quick_start_test.go +++ /dev/null @@ -1,40 +0,0 @@ -//go:build e2e -// +build e2e - -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package capkk - -import ( - "context" - - . "github.com/onsi/ginkgo" - capie2e "sigs.k8s.io/cluster-api/test/e2e" -) - -var _ = Describe("Cluster Creation using Cluster API quick-start test [PR-Blocking]", func() { - By("Creating single-node control plane with one worker node") - capie2e.QuickStartSpec(context.TODO(), func() capie2e.QuickStartSpecInput { - return capie2e.QuickStartSpecInput{ - E2EConfig: e2eConfig, - ClusterctlConfigPath: clusterctlConfigPath, - BootstrapClusterProxy: bootstrapClusterProxy, - ArtifactFolder: artifactFolder, - SkipCleanup: skipCleanup, - } - }) -}) diff --git a/test/e2e/suites/k3s/common.go b/test/e2e/suites/k3s/common.go deleted file mode 100644 index 1c4a2982..00000000 --- a/test/e2e/suites/k3s/common.go +++ /dev/null @@ -1,91 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k3s - -import ( - "context" - "fmt" - "path/filepath" - - . "github.com/onsi/ginkgo" - corev1 "k8s.io/api/core/v1" - "k8s.io/klog/v2" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/util" - - "github.com/kubesphere/kubekey/v3/test/e2e/framework" -) - -// Test suite constants for e2e config variables. -const ( - KubernetesVersion = "KUBERNETES_VERSION" - KubernetesVersionManagement = "KUBERNETES_VERSION_MANAGEMENT" - CNIPath = "CNI" - CNIResources = "CNI_RESOURCES" - IPFamily = "IP_FAMILY" -) - -// Byf is a wrapper around By that formats its arguments. -func Byf(format string, a ...interface{}) { - By(fmt.Sprintf(format, a...)) -} - -func setupSpecNamespace(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string) (*corev1.Namespace, context.CancelFunc) { - Byf("Creating a namespace for hosting the %q test spec", specName) - namespace, cancelWatches := framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ - Creator: clusterProxy.GetClient(), - ClientSet: clusterProxy.GetClientSet(), - Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), - LogFolder: filepath.Join(artifactFolder, "clusters", clusterProxy.GetName()), - }) - - return namespace, cancelWatches -} - -func dumpSpecResourcesAndCleanup(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string, namespace *corev1.Namespace, cancelWatches context.CancelFunc, cluster *clusterv1.Cluster, intervalsGetter func(spec, key string) []interface{}, skipCleanup bool) { - Byf("Dumping logs from the %q workload cluster", cluster.Name) - - // Dump all the logs from the workload cluster before deleting them. - clusterProxy.CollectWorkloadClusterLogs(ctx, cluster.Namespace, cluster.Name, filepath.Join(artifactFolder, "clusters", cluster.Name)) - - Byf("Dumping all the Cluster API resources in the %q namespace", namespace.Name) - - // Dump all Cluster API related resources to artifacts before deleting them. - framework.DumpAllResources(ctx, framework.DumpAllResourcesInput{ - Lister: clusterProxy.GetClient(), - Namespace: namespace.Name, - LogPath: filepath.Join(artifactFolder, "clusters", clusterProxy.GetName(), "resources"), - }) - - if !skipCleanup { - Byf("Deleting cluster %s", klog.KObj(cluster)) - // While https://github.com/kubernetes-sigs/cluster-api/issues/2955 is addressed in future iterations, there is a chance - // that cluster variable is not set even if the cluster exists, so we are calling DeleteAllClustersAndWait - // instead of DeleteClusterAndWait - framework.DeleteAllClustersAndWait(ctx, framework.DeleteAllClustersAndWaitInput{ - Client: clusterProxy.GetClient(), - Namespace: namespace.Name, - }, intervalsGetter(specName, "wait-delete-cluster")...) - - Byf("Deleting namespace used for hosting the %q test spec", specName) - framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{ - Deleter: clusterProxy.GetClient(), - Name: namespace.Name, - }) - } - cancelWatches() -} diff --git a/test/e2e/suites/k3s/doc.go b/test/e2e/suites/k3s/doc.go deleted file mode 100644 index e11b301e..00000000 --- a/test/e2e/suites/k3s/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package k3s implements k3s end to end testing. -package k3s diff --git a/test/e2e/suites/k3s/e2e_k3s_suite_test.go b/test/e2e/suites/k3s/e2e_k3s_suite_test.go deleted file mode 100644 index 901bf3db..00000000 --- a/test/e2e/suites/k3s/e2e_k3s_suite_test.go +++ /dev/null @@ -1,294 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k3s - -import ( - "flag" - "fmt" - "os" - "path/filepath" - "strings" - "testing" - "time" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" - ctrl "sigs.k8s.io/controller-runtime" - - "github.com/kubesphere/kubekey/v3/test/e2e/framework" - "github.com/kubesphere/kubekey/v3/test/e2e/framework/bootstrap" - "github.com/kubesphere/kubekey/v3/test/e2e/framework/clusterctl" -) - -// Test suite flags. -var ( - // configPath is the path to the e2e config file. - configPath string - - // useExistingCluster instructs the test to use the current cluster instead of creating a new one (default discovery rules apply). - useExistingCluster bool - - // artifactFolder is the folder to store e2e test artifacts. - artifactFolder string - - // clusterctlConfig is the file which tests will use as a clusterctl config. - // If it is not set, a local clusterctl repository (including a clusterctl config) will be created automatically. - clusterctlConfig string - - // alsoLogToFile enables additional logging to the 'ginkgo-log.txt' file in the artifact folder. - // These logs also contain timestamps. - alsoLogToFile bool - - // skipCleanup prevents cleanup of test resources e.g. for debug purposes. - skipCleanup bool -) - -// Test suite global vars. -var ( - ctx = ctrl.SetupSignalHandler() - - // e2eConfig to be used for this test, read from configPath. - e2eConfig *clusterctl.E2EConfig - - // clusterctlConfigPath to be used for this test, created by generating a clusterctl local repository - // with the providers specified in the configPath. - clusterctlConfigPath string - - // bootstrapClusterProvider manages provisioning of the bootstrap cluster to be used for the e2e tests. - // Please note that provisioning will be skipped if e2e.use-existing-cluster is provided. - bootstrapClusterProvider bootstrap.ClusterProvider - - // bootstrapClusterProxy allows to interact with the bootstrap cluster to be used for the e2e tests. - bootstrapClusterProxy framework.ClusterProxy -) - -func init() { - flag.StringVar(&configPath, "e2e.config", "", "path to the e2e config file") - flag.StringVar(&artifactFolder, "e2e.artifacts-folder", "", "folder where e2e test artifact should be stored") - flag.BoolVar(&alsoLogToFile, "e2e.also-log-to-file", true, "if true, ginkgo logs are additionally written to the `ginkgo-log.txt` file in the artifacts folder (including timestamps)") - flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped") - flag.StringVar(&clusterctlConfig, "e2e.clusterctl-config", "", "file which tests will use as a clusterctl config. If it is not set, a local clusterctl repository (including a clusterctl config) will be created automatically.") - flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)") - - SetDefaultEventuallyTimeout(30 * time.Minute) - SetDefaultEventuallyPollingInterval(10 * time.Second) -} - -func TestE2E(t *testing.T) { - g := NewWithT(t) - - // If running in prow, make sure to use the artifacts folder that will be reported in test grid (ignoring the value provided by flag). - if prowArtifactFolder, exists := os.LookupEnv("ARTIFACTS"); exists { - artifactFolder = prowArtifactFolder - } - - // ensure the artifacts folder exists - g.Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid test suite argument. Can't create e2e.artifacts-folder %q", artifactFolder) //nolint:gosec - - RegisterFailHandler(Fail) - - if alsoLogToFile { - w, err := ginkgoextensions.EnableFileLogging(filepath.Join(artifactFolder, "ginkgo-log.txt")) - g.Expect(err).ToNot(HaveOccurred()) - defer w.Close() - } - - RunSpecs(t, "capkk-k3s-e2e") -} - -// Using a SynchronizedBeforeSuite for controlling how to create resources shared across ParallelNodes (~ginkgo threads). -// The local clusterctl repository & the bootstrap cluster are created once and shared across all the tests. -var _ = SynchronizedBeforeSuite(func() []byte { - // Before all ParallelNodes. - Expect(configPath).To(BeAnExistingFile(), "Invalid test suite argument. e2e.config should be an existing file.") - - By("Initializing a runtime.Scheme with all the GVK relevant for this test") - scheme := initScheme() - - Byf("Loading the e2e test configuration from %q", configPath) - e2eConfig = loadE2EConfig(configPath) - - if clusterctlConfig == "" { - Byf("Creating a clusterctl local repository into %q", artifactFolder) - clusterctlConfigPath = createClusterctlLocalRepository(e2eConfig, filepath.Join(artifactFolder, "repository")) - } else { - Byf("Using existing clusterctl config %q", clusterctlConfig) - clusterctlConfigPath = clusterctlConfig - } - - By("Setting up the bootstrap cluster") - bootstrapClusterProvider, bootstrapClusterProxy = setupBootstrapCluster(e2eConfig, scheme, useExistingCluster) - - By("Initializing the bootstrap cluster") - initBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder) - - return []byte( - strings.Join([]string{ - artifactFolder, - configPath, - clusterctlConfigPath, - bootstrapClusterProxy.GetKubeconfigPath(), - }, ","), - ) -}, func(data []byte) { - // Before each ParallelNode. - - parts := strings.Split(string(data), ",") - Expect(parts).To(HaveLen(4)) - - artifactFolder = parts[0] - configPath = parts[1] - clusterctlConfigPath = parts[2] - kubeconfigPath := parts[3] - - e2eConfig = loadE2EConfig(configPath) - bootstrapClusterProxy = framework.NewClusterProxy("bootstrap", kubeconfigPath, initScheme()) -}) - -// Using a SynchronizedAfterSuite for controlling how to delete resources shared across ParallelNodes (~ginkgo threads). -// The bootstrap cluster is shared across all the tests, so it should be deleted only after all ParallelNodes completes. -// The local clusterctl repository is preserved like everything else created into the artifact folder. -var _ = SynchronizedAfterSuite(func() { - // After each ParallelNode. -}, func() { - // After all ParallelNodes. - - By("Dumping logs from the bootstrap cluster") - dumpBootstrapClusterLogs(bootstrapClusterProxy) - - By("Tearing down the management cluster") - if !skipCleanup { - tearDown(bootstrapClusterProvider, bootstrapClusterProxy) - } -}) - -func initScheme() *runtime.Scheme { - sc := runtime.NewScheme() - framework.TryAddDefaultSchemes(sc) - return sc -} - -func loadE2EConfig(configPath string) *clusterctl.E2EConfig { - config := clusterctl.LoadE2EConfig(ctx, clusterctl.LoadE2EConfigInput{ConfigPath: configPath}) - Expect(config).ToNot(BeNil(), "Failed to load E2E config from %s", configPath) - - return config -} - -func createClusterctlLocalRepository(config *clusterctl.E2EConfig, repositoryFolder string) string { - createRepositoryInput := clusterctl.CreateRepositoryInput{ - E2EConfig: config, - RepositoryFolder: repositoryFolder, - } - - // Ensuring a CNI file is defined in the config and register a FileTransformation to inject the referenced file in place of the CNI_RESOURCES envSubst variable. - Expect(config.Variables).To(HaveKey(CNIPath), "Missing %s variable in the config", CNIPath) - cniPath := config.GetVariable(CNIPath) - Expect(cniPath).To(BeAnExistingFile(), "The %s variable should resolve to an existing file", CNIPath) - - createRepositoryInput.RegisterClusterResourceSetConfigMapTransformation(cniPath, CNIResources) - - clusterctlConfig := clusterctl.CreateRepository(ctx, createRepositoryInput) - Expect(clusterctlConfig).To(BeAnExistingFile(), "The clusterctl config file does not exists in the local repository %s", repositoryFolder) - return clusterctlConfig -} - -func setupBootstrapCluster(config *clusterctl.E2EConfig, scheme *runtime.Scheme, useExistingCluster bool) (bootstrap.ClusterProvider, framework.ClusterProxy) { - var clusterProvider bootstrap.ClusterProvider - kubeconfigPath := "" - if !useExistingCluster { - By("Creating the bootstrap cluster") - clusterProvider = bootstrap.CreateKindBootstrapClusterAndLoadImages(ctx, bootstrap.CreateKindBootstrapClusterAndLoadImagesInput{ - Name: config.ManagementClusterName, - KubernetesVersion: config.GetVariable(KubernetesVersionManagement), - RequiresDockerSock: config.HasDockerProvider(), - Images: config.Images, - IPFamily: config.GetVariable(IPFamily), - LogFolder: filepath.Join(artifactFolder, "kind"), - }) - Expect(clusterProvider).ToNot(BeNil(), "Failed to create a bootstrap cluster") - - kubeconfigPath = clusterProvider.GetKubeconfigPath() - Expect(kubeconfigPath).To(BeAnExistingFile(), "Failed to get the kubeconfig file for the bootstrap cluster") - } else { - By("Using an existing bootstrap cluster") - } - - clusterProxy := framework.NewClusterProxy("bootstrap", kubeconfigPath, scheme) - Expect(clusterProxy).ToNot(BeNil(), "Failed to get a bootstrap cluster proxy") - - return clusterProvider, clusterProxy -} - -func initBootstrapCluster(bootstrapClusterProxy framework.ClusterProxy, config *clusterctl.E2EConfig, clusterctlConfig, artifactFolder string) { - clusterctl.InitManagementClusterAndWatchControllerLogs(ctx, clusterctl.InitManagementClusterAndWatchControllerLogsInput{ - ClusterProxy: bootstrapClusterProxy, - ClusterctlConfigPath: clusterctlConfig, - InfrastructureProviders: config.InfrastructureProviders(), - LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), - }, config.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) -} - -func dumpBootstrapClusterLogs(bootstrapClusterProxy framework.ClusterProxy) { - if bootstrapClusterProxy == nil { - return - } - - clusterLogCollector := bootstrapClusterProxy.GetLogCollector() - if clusterLogCollector == nil { - return - } - - nodes, err := bootstrapClusterProxy.GetClientSet().CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - if err != nil { - fmt.Printf("Failed to get nodes for the bootstrap cluster: %v\n", err) - return - } - - for i := range nodes.Items { - nodeName := nodes.Items[i].GetName() - err = clusterLogCollector.CollectMachineLog( - ctx, - bootstrapClusterProxy.GetClient(), - // The bootstrap cluster is not expected to be a CAPI cluster, so in order to re-use the logCollector, - // we create a fake machine that wraps the node. - // NOTE: This assumes a naming convention between machines and nodes, which e.g. applies to the bootstrap clusters generated with kind. - // This might not work if you are using an existing bootstrap cluster provided by other means. - &clusterv1.Machine{ - Spec: clusterv1.MachineSpec{ClusterName: nodeName}, - ObjectMeta: metav1.ObjectMeta{Name: nodeName}, - }, - filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName(), "machines", nodeName), - ) - if err != nil { - fmt.Printf("Failed to get logs for the bootstrap cluster node %s: %v\n", nodeName, err) - } - } -} - -func tearDown(bootstrapClusterProvider bootstrap.ClusterProvider, bootstrapClusterProxy framework.ClusterProxy) { - if bootstrapClusterProxy != nil { - bootstrapClusterProxy.Dispose(ctx) - } - if bootstrapClusterProvider != nil { - bootstrapClusterProvider.Dispose(ctx) - } -} diff --git a/test/e2e/suites/k3s/quick_start.go b/test/e2e/suites/k3s/quick_start.go deleted file mode 100644 index d6ad8f9f..00000000 --- a/test/e2e/suites/k3s/quick_start.go +++ /dev/null @@ -1,113 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k3s - -import ( - "context" - "fmt" - "os" - "path/filepath" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - "k8s.io/utils/pointer" - "sigs.k8s.io/cluster-api/util" - - "github.com/kubesphere/kubekey/v3/test/e2e/framework" - "github.com/kubesphere/kubekey/v3/test/e2e/framework/clusterctl" -) - -// QuickStartSpecInput is the input for QuickStartSpec. -type QuickStartSpecInput struct { - E2EConfig *clusterctl.E2EConfig - ClusterctlConfigPath string - BootstrapClusterProxy framework.ClusterProxy - ArtifactFolder string - SkipCleanup bool - ControlPlaneWaiters clusterctl.ControlPlaneWaiters - - // Flavor, if specified is the template flavor used to create the cluster for testing. - // If not specified, and the e2econfig variable IPFamily is IPV6, then "ipv6" is used, - // otherwise the default flavor is used. - Flavor *string -} - -// QuickStartSpec implements a spec that mimics the operation described in the Cluster API quick start, that is -// creating a workload cluster. -// This test is meant to provide a first, fast signal to detect regression; it is recommended to use it as a PR blocker test. -// NOTE: This test works with Clusters with and without ClusterClass. -func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput) { - var ( - specName = "quick-start" - input QuickStartSpecInput - namespace *corev1.Namespace - cancelWatches context.CancelFunc - clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult - ) - - BeforeEach(func() { - Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) - input = inputGetter() - Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) - Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) - Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) - Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) - - Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) - - // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. - namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) - clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) - }) - - It("Should create a workload cluster", func() { - By("Creating a workload cluster") - - flavor := clusterctl.DefaultFlavor - if input.Flavor != nil { - flavor = *input.Flavor - } - - clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ - ClusterProxy: input.BootstrapClusterProxy, - ConfigCluster: clusterctl.ConfigClusterInput{ - LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), - ClusterctlConfigPath: input.ClusterctlConfigPath, - KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), - InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, - Flavor: flavor, - Namespace: namespace.Name, - ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), - KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), - ControlPlaneMachineCount: pointer.Int64(1), - WorkerMachineCount: pointer.Int64(1), - }, - ControlPlaneWaiters: input.ControlPlaneWaiters, - WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), - WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), - WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), - }, clusterResources) - - By("PASSED!") - }) - - AfterEach(func() { - // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. - dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) - }) -} diff --git a/test/e2e/suites/k3s/quick_start_test.go b/test/e2e/suites/k3s/quick_start_test.go deleted file mode 100644 index 7cc9954f..00000000 --- a/test/e2e/suites/k3s/quick_start_test.go +++ /dev/null @@ -1,39 +0,0 @@ -//go:build e2e -// +build e2e - -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package k3s - -import ( - "context" - - . "github.com/onsi/ginkgo" -) - -var _ = Describe("Cluster Creation using Cluster API quick-start test [PR-Blocking]", func() { - By("Creating single-node control plane with one worker node") - QuickStartSpec(context.TODO(), func() QuickStartSpecInput { - return QuickStartSpecInput{ - E2EConfig: e2eConfig, - ClusterctlConfigPath: clusterctlConfigPath, - BootstrapClusterProxy: bootstrapClusterProxy, - ArtifactFolder: artifactFolder, - SkipCleanup: skipCleanup, - } - }) -}) diff --git a/util/collections/doc.go b/util/collections/doc.go deleted file mode 100644 index eb39bc62..00000000 --- a/util/collections/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package collections implements collection utilities. -package collections diff --git a/util/collections/helpers.go b/util/collections/helpers.go deleted file mode 100644 index a79f7284..00000000 --- a/util/collections/helpers.go +++ /dev/null @@ -1,45 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package collections - -import ( - "context" - - "github.com/pkg/errors" - "sigs.k8s.io/controller-runtime/pkg/client" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" -) - -// GetFilteredKKInstancesForKKCluster returns a list of kkInstances that can be filtered or not. -// If no filter is supplied then all kkInstances associated with the target kkCluster are returned. -func GetFilteredKKInstancesForKKCluster(ctx context.Context, c client.Reader, kkCluster *infrav1.KKCluster, filters ...Func) (KKInstances, error) { - ml := &infrav1.KKInstanceList{} - if err := c.List( - ctx, - ml, - client.InNamespace(kkCluster.Namespace), - client.MatchingLabels{ - infrav1.KKClusterLabelName: kkCluster.Name, - }, - ); err != nil { - return nil, errors.Wrap(err, "failed to list machines") - } - - kkInstances := FromKKInstanceList(ml) - return kkInstances.Filter(filters...), nil -} diff --git a/util/collections/kkinstance_collection.go b/util/collections/kkinstance_collection.go deleted file mode 100644 index 51aa662f..00000000 --- a/util/collections/kkinstance_collection.go +++ /dev/null @@ -1,179 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package collections - -import ( - "sort" - - "sigs.k8s.io/cluster-api/util/conditions" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" -) - -// KKInstances is a set of KKInstances. -type KKInstances map[string]*infrav1.KKInstance - -// kkInstancesByCreationTimestamp sorts a list of KKInstance by creation timestamp, using their names as a tie breaker. -type kkInstancesByCreationTimestamp []*infrav1.KKInstance - -func (o kkInstancesByCreationTimestamp) Len() int { return len(o) } -func (o kkInstancesByCreationTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] } -func (o kkInstancesByCreationTimestamp) Less(i, j int) bool { - if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) { - return o[i].Name < o[j].Name - } - return o[i].CreationTimestamp.Before(&o[j].CreationTimestamp) -} - -// New creates an empty KKInstances. -func New() KKInstances { - return make(KKInstances) -} - -// FromKKInstances creates a KKInstance from a list of values. -func FromKKInstances(kkInstances ...*infrav1.KKInstance) KKInstances { - ss := make(KKInstances, len(kkInstances)) - ss.Insert(kkInstances...) - return ss -} - -// FromKKInstanceList creates a KKInstances from the given KKInstanceList. -func FromKKInstanceList(kkInstanceList *infrav1.KKInstanceList) KKInstances { - ss := make(KKInstances, len(kkInstanceList.Items)) - for i := range kkInstanceList.Items { - ss.Insert(&kkInstanceList.Items[i]) - } - return ss -} - -// ToKKInstanceList creates a KKInstanceList from the given KKInstances. -func ToKKInstanceList(kkInstances KKInstances) infrav1.KKInstanceList { - ml := infrav1.KKInstanceList{} - for _, m := range kkInstances { - ml.Items = append(ml.Items, *m) - } - return ml -} - -// Insert adds items to the set. -func (s KKInstances) Insert(kkInstances ...*infrav1.KKInstance) { - for i := range kkInstances { - if kkInstances[i] != nil { - m := kkInstances[i] - s[m.Name] = m - } - } -} - -// Difference returns a copy without KKInstances that are in the given collection. -func (s KKInstances) Difference(kkInstances KKInstances) KKInstances { - return s.Filter(func(m *infrav1.KKInstance) bool { - _, found := kkInstances[m.Name] - return !found - }) -} - -// SortedByCreationTimestamp returns the KKInstances sorted by creation timestamp. -func (s KKInstances) SortedByCreationTimestamp() []*infrav1.KKInstance { - res := make(kkInstancesByCreationTimestamp, 0, len(s)) - for _, value := range s { - res = append(res, value) - } - sort.Sort(res) - return res -} - -// UnsortedList returns the slice with contents in random order. -func (s KKInstances) UnsortedList() []*infrav1.KKInstance { - res := make([]*infrav1.KKInstance, 0, len(s)) - for _, value := range s { - res = append(res, value) - } - return res -} - -// Len returns the size of the set. -func (s KKInstances) Len() int { - return len(s) -} - -// newFilteredKKInstanceCollection creates a KKInstances from a filtered list of values. -func newFilteredKKInstanceCollection(filter Func, kkInstances ...*infrav1.KKInstance) KKInstances { - ss := make(KKInstances, len(kkInstances)) - for i := range kkInstances { - m := kkInstances[i] - if filter(m) { - ss.Insert(m) - } - } - return ss -} - -// Filter returns a KKInstances containing only the KKInstances that match all of the given KKInstanceFilters. -func (s KKInstances) Filter(filters ...Func) KKInstances { - return newFilteredKKInstanceCollection(And(filters...), s.UnsortedList()...) -} - -// AnyFilter returns a KKInstances containing only the KKInstances that match any of the given KKInstanceFilters. -func (s KKInstances) AnyFilter(filters ...Func) KKInstances { - return newFilteredKKInstanceCollection(Or(filters...), s.UnsortedList()...) -} - -// Oldest returns the KKInstances with the oldest CreationTimestamp. -func (s KKInstances) Oldest() *infrav1.KKInstance { - if len(s) == 0 { - return nil - } - return s.SortedByCreationTimestamp()[0] -} - -// Newest returns the KKInstance with the most recent CreationTimestamp. -func (s KKInstances) Newest() *infrav1.KKInstance { - if len(s) == 0 { - return nil - } - return s.SortedByCreationTimestamp()[len(s)-1] -} - -// DeepCopy returns a deep copy. -func (s KKInstances) DeepCopy() KKInstances { - result := make(KKInstances, len(s)) - for _, m := range s { - result.Insert(m.DeepCopy()) - } - return result -} - -// ConditionGetters returns the slice with KKInstances converted into conditions.Getter. -func (s KKInstances) ConditionGetters() []conditions.Getter { - res := make([]conditions.Getter, 0, len(s)) - for _, v := range s { - value := *v - res = append(res, &value) - } - return res -} - -// Names returns a slice of the names of each KKInstance in the collection. -// Useful for logging and test assertions. -func (s KKInstances) Names() []string { - names := make([]string, 0, s.Len()) - for _, m := range s { - names = append(names, m.Name) - } - return names -} diff --git a/util/collections/kkinstance_filters.go b/util/collections/kkinstance_filters.go deleted file mode 100644 index 977f97a0..00000000 --- a/util/collections/kkinstance_filters.go +++ /dev/null @@ -1,107 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package collections - -import ( - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/selection" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/controller-runtime/pkg/client" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" -) - -// Func is the functon definition for a filter. -type Func func(kkInstance *infrav1.KKInstance) bool - -// And returns a filter that returns true if all of the given filters returns true. -func And(filters ...Func) Func { - return func(kkInstance *infrav1.KKInstance) bool { - for _, f := range filters { - if !f(kkInstance) { - return false - } - } - return true - } -} - -// Or returns a filter that returns true if any of the given filters returns true. -func Or(filters ...Func) Func { - return func(kkInstance *infrav1.KKInstance) bool { - for _, f := range filters { - if f(kkInstance) { - return true - } - } - return false - } -} - -// Not returns a filter that returns the opposite of the given filter. -func Not(mf Func) Func { - return func(kkInstance *infrav1.KKInstance) bool { - return !mf(kkInstance) - } -} - -// OwnedKKInstances returns a filter to find all kkInstances owned by specified owner. -// Usage: GetFilteredKKInstancesForKKCluster(ctx, client, cluster, OwnedKKInstances(controlPlane)). -func OwnedKKInstances(owner client.Object) func(kkInstance *infrav1.KKInstance) bool { - return func(kkInstance *infrav1.KKInstance) bool { - if kkInstance == nil { - return false - } - return util.IsOwnedByObject(kkInstance, owner) - } -} - -// ControlPlaneKKInstances returns a filter to find all control plane KKInstance for a cluster, regardless of ownership. -// Usage: GetFilteredKKInstancesForKKCluster(ctx, client, cluster, ControlPlaneKKInstances(cluster.Name)). -func ControlPlaneKKInstances(clusterName string) func(kkInstance *infrav1.KKInstance) bool { - selector := ControlPlaneSelectorForCluster(clusterName) - return func(kkInstance *infrav1.KKInstance) bool { - if kkInstance == nil { - return false - } - return selector.Matches(labels.Set(kkInstance.Labels)) - } -} - -// ActiveKKInstances returns a filter to find all active kkinstances. -// Usage: GetFilteredKKInstancesForKKCluster(ctx, client, cluster, ActiveKKInstances). -func ActiveKKInstances(kkInstance *infrav1.KKInstance) bool { - if kkInstance == nil { - return false - } - return kkInstance.DeletionTimestamp.IsZero() -} - -// ControlPlaneSelectorForCluster returns the label selector necessary to get control plane KKInstance for a given cluster. -func ControlPlaneSelectorForCluster(clusterName string) labels.Selector { - must := func(r *labels.Requirement, err error) labels.Requirement { - if err != nil { - panic(err) - } - return *r - } - return labels.NewSelector().Add( - must(labels.NewRequirement(clusterv1.ClusterLabelName, selection.Equals, []string{clusterName})), - must(labels.NewRequirement(clusterv1.MachineControlPlaneLabelName, selection.Exists, []string{})), - ) -} diff --git a/util/log/doc.go b/util/log/doc.go deleted file mode 100644 index af33e743..00000000 --- a/util/log/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package log provides log utils. -package log diff --git a/util/log/log.go b/util/log/log.go deleted file mode 100644 index 01464cc1..00000000 --- a/util/log/log.go +++ /dev/null @@ -1,105 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package log provides log utils. -package log - -import ( - "context" - - "github.com/go-logr/logr" - "github.com/pkg/errors" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/klog/v2" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// AddOwners adds the owners of an Object based on OwnerReferences as k/v pairs to the logger in ctx. -// Note: If an owner is a MachineSet we also add the owners from the MachineSet OwnerReferences. -func AddOwners(ctx context.Context, c client.Client, obj metav1.Object) (context.Context, logr.Logger, error) { - log := ctrl.LoggerFrom(ctx) - - owners, err := getOwners(ctx, c, obj) - if err != nil { - return nil, logr.Logger{}, errors.Wrapf(err, "failed to add object hierarchy to logger") - } - - // Add owners as k/v pairs. - keysAndValues := []interface{}{} - addedKinds := sets.NewString() - for _, owner := range owners { - // Don't add duplicate kinds. - if addedKinds.Has(owner.Kind) { - continue - } - - keysAndValues = append(keysAndValues, owner.Kind, klog.KRef(owner.Namespace, owner.Name)) - addedKinds.Insert(owner.Kind) - } - log = log.WithValues(keysAndValues...) - - ctx = ctrl.LoggerInto(ctx, log) - return ctx, log, nil -} - -// owner represents an owner of an object. -type owner struct { - Kind string - Name string - Namespace string -} - -// getOwners returns owners of an Object based on OwnerReferences. -// Note: If an owner is a MachineSet we also return the owners from the MachineSet OwnerReferences. -func getOwners(ctx context.Context, c client.Client, obj metav1.Object) ([]owner, error) { - owners := []owner{} - for _, ownerRef := range obj.GetOwnerReferences() { - owners = append(owners, owner{ - Kind: ownerRef.Kind, - Namespace: obj.GetNamespace(), - Name: ownerRef.Name, - }) - - // continue if the ownerRef does not point to a MachineSet. - if ownerRef.Kind != "MachineSet" { - continue - } - - // get owners of the MachineSet. - var ms clusterv1.MachineSet - if err := c.Get(ctx, client.ObjectKey{Namespace: obj.GetNamespace(), Name: ownerRef.Name}, &ms); err != nil { - // continue if the MachineSet doesn't exist. - if apierrors.IsNotFound(err) { - continue - } - return nil, errors.Wrapf(err, "failed to get owners: failed to get MachineSet %s", klog.KRef(obj.GetNamespace(), ownerRef.Name)) - } - - for _, ref := range ms.GetOwnerReferences() { - owners = append(owners, owner{ - Kind: ref.Kind, - Namespace: obj.GetNamespace(), - Name: ref.Name, - }) - } - } - - return owners, nil -} diff --git a/util/log/log_test.go b/util/log/log_test.go deleted file mode 100644 index 7078ebb3..00000000 --- a/util/log/log_test.go +++ /dev/null @@ -1,215 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package log - -import ( - "context" - "testing" - - "github.com/go-logr/logr" - . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/klog/v2" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/log" -) - -func Test_AddObjectHierarchy(t *testing.T) { - g := NewWithT(t) - - scheme := runtime.NewScheme() - g.Expect(clusterv1.AddToScheme(scheme)).To(Succeed()) - - md := &clusterv1.MachineSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: clusterv1.GroupVersion.String(), - Kind: "MachineDeployment", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: metav1.NamespaceDefault, - Name: "development-3961-md-0-l4zn6", - }, - } - mdOwnerRef := metav1.OwnerReference{ - APIVersion: md.APIVersion, - Kind: md.Kind, - Name: md.Name, - } - - ms := &clusterv1.MachineSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: clusterv1.GroupVersion.String(), - Kind: "MachineSet", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: metav1.NamespaceDefault, - Name: "development-3961-md-0-l4zn6-758c9b7677", - OwnerReferences: []metav1.OwnerReference{mdOwnerRef}, - }, - } - msOwnerRef := metav1.OwnerReference{ - APIVersion: ms.APIVersion, - Kind: ms.Kind, - Name: ms.Name, - } - - tests := []struct { - name string - obj metav1.Object - objects []client.Object - expectedKeysAndValues []interface{} - }{ - { - name: "MachineSet owning Machine is added", - // MachineSet does not exist in Kubernetes so only MachineSet is added - // and MachineDeployment is not. - obj: &clusterv1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - OwnerReferences: []metav1.OwnerReference{msOwnerRef}, - Namespace: metav1.NamespaceDefault, - }, - }, - expectedKeysAndValues: []interface{}{ - "MachineSet", - klog.ObjectRef{Namespace: ms.Namespace, Name: ms.Name}, - }, - }, - { - name: "MachineDeployment and MachineSet owning Machine is added", - obj: &clusterv1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - OwnerReferences: []metav1.OwnerReference{msOwnerRef}, - Namespace: metav1.NamespaceDefault, - }, - }, - objects: []client.Object{ms}, - expectedKeysAndValues: []interface{}{ - "MachineSet", - klog.ObjectRef{Namespace: ms.Namespace, Name: ms.Name}, - "MachineDeployment", - klog.ObjectRef{Namespace: md.Namespace, Name: md.Name}, - }, - }, - { - name: "MachineDeployment owning MachineSet is added", - obj: &clusterv1.MachineSet{ - ObjectMeta: metav1.ObjectMeta{ - OwnerReferences: []metav1.OwnerReference{mdOwnerRef}, - Namespace: metav1.NamespaceDefault, - }, - }, - expectedKeysAndValues: []interface{}{ - "MachineDeployment", - klog.ObjectRef{Namespace: md.Namespace, Name: md.Name}, - }, - }, - { - name: "K3sControlPlane and Machine owning DockerMachine are added", - obj: &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "infrastructure.cluster.x-k8s.io/v1beta1", - "kind": "DockerMachine", - "metadata": map[string]interface{}{ - "ownerReferences": []interface{}{ - map[string]interface{}{ - "apiVersion": clusterv1.GroupVersion.String(), - "kind": "Machine", - "name": "development-3961-4flkb-gzxnb", - }, - map[string]interface{}{ - "apiVersion": clusterv1.GroupVersion.String(), - "kind": "K3sControlPlane", - "name": "development-3961-4flkb", - }, - }, - "namespace": metav1.NamespaceDefault, - }, - }, - }, - expectedKeysAndValues: []interface{}{ - "Machine", - klog.ObjectRef{Namespace: metav1.NamespaceDefault, Name: "development-3961-4flkb-gzxnb"}, - "K3sControlPlane", - klog.ObjectRef{Namespace: metav1.NamespaceDefault, Name: "development-3961-4flkb"}, - }, - }, - { - name: "Duplicate Cluster ownerRef should be deduplicated", - obj: &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "infrastructure.cluster.x-k8s.io/v1beta1", - "kind": "DockerCluster", - "metadata": map[string]interface{}{ - "ownerReferences": []interface{}{ - map[string]interface{}{ - "apiVersion": clusterv1.GroupVersion.String(), - "kind": "Cluster", - "name": "development-3961", - }, - map[string]interface{}{ - "apiVersion": clusterv1.GroupVersion.String(), - "kind": "Cluster", - "name": "development-3961", - }, - }, - "namespace": metav1.NamespaceDefault, - }, - }, - }, - expectedKeysAndValues: []interface{}{ - "Cluster", - klog.ObjectRef{Namespace: metav1.NamespaceDefault, Name: "development-3961"}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - - c := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(tt.objects...). - Build() - - // Create fake log sink so we can later verify the added k/v pairs. - ctx := ctrl.LoggerInto(context.Background(), logr.New(&fakeLogSink{})) - - _, logger, err := AddOwners(ctx, c, tt.obj) - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(logger.GetSink().(fakeLogSink).keysAndValues).To(Equal(tt.expectedKeysAndValues)) - }) - } -} - -type fakeLogSink struct { - // Embedding NullLogSink so we don't have to implement all funcs - // of the LogSink interface. - log.NullLogSink - keysAndValues []interface{} -} - -// WithValues stores keysAndValues so we can later check if the -// right keysAndValues have been added. -func (f fakeLogSink) WithValues(keysAndValues ...interface{}) logr.LogSink { - f.keysAndValues = keysAndValues - return f -} diff --git a/util/osrelease/osrelease.go b/util/osrelease/osrelease.go deleted file mode 100644 index 89e8f3d9..00000000 --- a/util/osrelease/osrelease.go +++ /dev/null @@ -1,158 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package osrelease is to parse a os release file content. -package osrelease - -import ( - "bufio" - "bytes" - "strings" - - "github.com/pkg/errors" -) - -const ( - // EtcOsRelease is the path of os-release file. - EtcOsRelease string = "/etc/os-release" - // DebianID is the identifier used by the Debian operating system. - DebianID = "debian" - // FedoraID is the identifier used by the Fedora operating system. - FedoraID = "fedora" - // UbuntuID is the identifier used by the Ubuntu operating system. - UbuntuID = "ubuntu" - // RhelID is the identifier used by the Rhel operating system. - RhelID = "rhel" - // CentosID is the identifier used by the Centos operating system. - CentosID = "centos" -) - -// Data exposes the most common identification parameters. -type Data struct { - ID string - IDLike string - Name string - PrettyName string - Version string - VersionID string -} - -// Parse is to parse a os release file content. -func Parse(content string) (data *Data) { - data = new(Data) - lines, err := parseString(content) - if err != nil { - return - } - - info := make(map[string]string) - for _, v := range lines { - key, value, err := parseLine(v) - if err == nil { - info[key] = value - } - } - data.ID = info["ID"] - data.IDLike = info["ID_LIKE"] - data.Name = info["NAME"] - data.PrettyName = info["PRETTY_NAME"] - data.Version = info["VERSION"] - data.VersionID = info["VERSION_ID"] - return -} - -func parseString(content string) (lines []string, err error) { - in := bytes.NewBufferString(content) - reader := bufio.NewReader(in) - scanner := bufio.NewScanner(reader) - - for scanner.Scan() { - lines = append(lines, scanner.Text()) - } - return lines, scanner.Err() -} - -func parseLine(line string) (string, string, error) { - // skip empty lines - if line == "" { - return "", "", errors.New("Skipping: zero-length") - } - - // skip comments - if line[0] == '#' { - return "", "", errors.New("Skipping: comment") - } - - // try to split string at the first '=' - splitString := strings.SplitN(line, "=", 2) - if len(splitString) != 2 { - return "", "", errors.New("Can not extract key=value") - } - - // trim white space from key and value - key := splitString[0] - key = strings.Trim(key, " ") - value := splitString[1] - value = strings.Trim(value, " ") - - // Handle double quotes - if strings.ContainsAny(value, `"`) { - first := value[0:1] - last := value[len(value)-1:] - - if first == last && strings.ContainsAny(first, `"'`) { - value = strings.TrimPrefix(value, `'`) - value = strings.TrimPrefix(value, `"`) - value = strings.TrimSuffix(value, `'`) - value = strings.TrimSuffix(value, `"`) - } - } - - // expand anything else that could be escaped - value = strings.Replace(value, `\"`, `"`, -1) - value = strings.Replace(value, `\$`, `$`, -1) - value = strings.Replace(value, `\\`, `\`, -1) - value = strings.Replace(value, "\\`", "`", -1) - value = strings.TrimRight(value, "\r\n") - value = strings.TrimLeft(value, "\"") - value = strings.TrimRight(value, "\"") - return key, value, nil -} - -// IsLikeDebian will return true for Debian and any other related OS, such as Ubuntu. -func (d *Data) IsLikeDebian() bool { - return d.ID == DebianID || strings.Contains(d.IDLike, DebianID) -} - -// IsLikeFedora will return true for Fedora and any other related OS, such as CentOS or RHEL. -func (d *Data) IsLikeFedora() bool { - return d.ID == FedoraID || strings.Contains(d.IDLike, FedoraID) -} - -// IsUbuntu will return true for Ubuntu OS. -func (d *Data) IsUbuntu() bool { - return d.ID == UbuntuID -} - -// IsRHEL will return true for RHEL OS. -func (d *Data) IsRHEL() bool { - return d.ID == RhelID -} - -// IsCentOS will return true for CentOS. -func (d *Data) IsCentOS() bool { - return d.ID == CentosID -} diff --git a/util/osrelease/osrelease_test.go b/util/osrelease/osrelease_test.go deleted file mode 100644 index 34c9735f..00000000 --- a/util/osrelease/osrelease_test.go +++ /dev/null @@ -1,107 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package osrelease - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "reflect" - "testing" -) - -var testResult = []struct { - name string - data *Data -}{ - { - name: "ubuntu2204", - data: &Data{ - ID: "ubuntu", - IDLike: "debian", - Name: "Ubuntu", - PrettyName: "Ubuntu 22.04 LTS", - Version: "22.04 (Jammy Jellyfish)", - VersionID: "22.04", - }, - }, - { - name: "ubuntu2004", - data: &Data{ - ID: "ubuntu", - IDLike: "debian", - Name: "Ubuntu", - PrettyName: "Ubuntu 20.04.4 LTS", - Version: "20.04.4 LTS (Focal Fossa)", - VersionID: "20.04", - }, - }, - { - name: "ubuntu1804", - data: &Data{ - ID: "ubuntu", - IDLike: "debian", - Name: "Ubuntu", - PrettyName: "Ubuntu 18.04.2 LTS", - Version: "18.04.2 LTS (Bionic Beaver)", - VersionID: "18.04", - }, - }, - { - name: "centos7", - data: &Data{ - ID: "centos", - IDLike: "rhel fedora", - Name: "CentOS Linux", - PrettyName: "CentOS Linux 7 (Core)", - Version: "7 (Core)", - VersionID: "7", - }, - }, -} - -func TestParse(t *testing.T) { - for _, tt := range testResult { - t.Run(tt.name, func(t *testing.T) { - bs, err := os.ReadFile(filepath.Join("test", tt.name)) - if err != nil { - t.Errorf("cannot read datafile: %v", err) - } - info := Parse(string(bs)) - if !reflect.DeepEqual(tt.data, info) { - t.Errorf("Parse() got = %v, want %v", info, tt.data) - } - }) - } -} - -func TestParseByCommand(t *testing.T) { - for _, tt := range testResult { - t.Run(tt.name, func(t *testing.T) { - c := fmt.Sprintf("cat %s", filepath.Join("test", tt.name)) - bs, err := exec.Command("/bin/bash", "-c", c).Output() //nolint:gosec - if err != nil { - t.Errorf("cannot cat datafile: %v", err) - } - info := Parse(string(bs)) - if !reflect.DeepEqual(tt.data, info) { - t.Errorf("Parse() got = %v, want %v", info, tt.data) - } - }) - } -} diff --git a/util/osrelease/test/centos7 b/util/osrelease/test/centos7 deleted file mode 100644 index c276e3ae..00000000 --- a/util/osrelease/test/centos7 +++ /dev/null @@ -1,15 +0,0 @@ -NAME="CentOS Linux" -VERSION="7 (Core)" -ID="centos" -ID_LIKE="rhel fedora" -VERSION_ID="7" -PRETTY_NAME="CentOS Linux 7 (Core)" -ANSI_COLOR="0;31" -CPE_NAME="cpe:/o:centos:centos:7" -HOME_URL="https://www.centos.org/" -BUG_REPORT_URL="https://bugs.centos.org/" - -CENTOS_MANTISBT_PROJECT="CentOS-7" -CENTOS_MANTISBT_PROJECT_VERSION="7" -REDHAT_SUPPORT_PRODUCT="centos" -REDHAT_SUPPORT_PRODUCT_VERSION="7" diff --git a/util/osrelease/test/rhel7 b/util/osrelease/test/rhel7 deleted file mode 100644 index 62990f12..00000000 --- a/util/osrelease/test/rhel7 +++ /dev/null @@ -1,17 +0,0 @@ -NAME="Red Hat Enterprise Linux Server" -VERSION="7.6 (Maipo)" -ID="rhel" -ID_LIKE="fedora" -VARIANT="Server" -VARIANT_ID="server" -VERSION_ID="7.6" -PRETTY_NAME="Red Hat Enterprise Linux 7.6" -ANSI_COLOR="0;31" -CPE_NAME="cpe:/o:redhat:enterprise_linux:7.6:GA:server" -HOME_URL="https://www.redhat.com/" -BUG_REPORT_URL="https://bugzilla.redhat.com/" - -REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 7" -REDHAT_BUGZILLA_PRODUCT_VERSION=7.6 -REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" -REDHAT_SUPPORT_PRODUCT_VERSION="7.6" \ No newline at end of file diff --git a/util/osrelease/test/ubuntu1804 b/util/osrelease/test/ubuntu1804 deleted file mode 100644 index 5769a040..00000000 --- a/util/osrelease/test/ubuntu1804 +++ /dev/null @@ -1,12 +0,0 @@ -NAME="Ubuntu" -VERSION="18.04.2 LTS (Bionic Beaver)" -ID=ubuntu -ID_LIKE=debian -PRETTY_NAME="Ubuntu 18.04.2 LTS" -VERSION_ID="18.04" -HOME_URL="https://www.ubuntu.com/" -SUPPORT_URL="https://help.ubuntu.com/" -BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" -PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" -VERSION_CODENAME=bionic -UBUNTU_CODENAME=bionic \ No newline at end of file diff --git a/util/osrelease/test/ubuntu2004 b/util/osrelease/test/ubuntu2004 deleted file mode 100644 index c5fd8398..00000000 --- a/util/osrelease/test/ubuntu2004 +++ /dev/null @@ -1,12 +0,0 @@ -NAME="Ubuntu" -VERSION="20.04.4 LTS (Focal Fossa)" -ID=ubuntu -ID_LIKE=debian -PRETTY_NAME="Ubuntu 20.04.4 LTS" -VERSION_ID="20.04" -HOME_URL="https://www.ubuntu.com/" -SUPPORT_URL="https://help.ubuntu.com/" -BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" -PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" -VERSION_CODENAME=focal -UBUNTU_CODENAME=focal \ No newline at end of file diff --git a/util/osrelease/test/ubuntu2204 b/util/osrelease/test/ubuntu2204 deleted file mode 100644 index 7cbc72d3..00000000 --- a/util/osrelease/test/ubuntu2204 +++ /dev/null @@ -1,12 +0,0 @@ -PRETTY_NAME="Ubuntu 22.04 LTS" -NAME="Ubuntu" -VERSION_ID="22.04" -VERSION="22.04 (Jammy Jellyfish)" -VERSION_CODENAME=jammy -ID=ubuntu -ID_LIKE=debian -HOME_URL="https://www.ubuntu.com/" -SUPPORT_URL="https://help.ubuntu.com/" -BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" -PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" -UBUNTU_CODENAME=jammy \ No newline at end of file diff --git a/util/secret/certificates.go b/util/secret/certificates.go deleted file mode 100644 index 3c0338b7..00000000 --- a/util/secret/certificates.go +++ /dev/null @@ -1,400 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package secret - -import ( - "context" - "crypto/rand" - "crypto/rsa" - "crypto/sha256" - "crypto/x509" - "crypto/x509/pkix" - "encoding/hex" - "math/big" - "path/filepath" - "strings" - "time" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/util/cert" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" - "sigs.k8s.io/cluster-api/util/certs" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - rootOwnerValue = "root:root" - - // DefaultCertificatesDir is the default directory where k3s certificates are stored - DefaultCertificatesDir = "/var/lib/rancher/k3s/server/tls" -) - -var ( - // ErrMissingCertificate is an error indicating a certificate is entirely missing - ErrMissingCertificate = errors.New("missing certificate") - - // ErrMissingCrt is an error indicating the crt file is missing from the certificate - ErrMissingCrt = errors.New("missing crt data") - - // ErrMissingKey is an error indicating the key file is missing from the certificate - ErrMissingKey = errors.New("missing key data") -) - -// Certificates are the certificates necessary to bootstrap a cluster. -type Certificates []*Certificate - -// NewCertificatesForInitialControlPlane returns a list of certificates configured for a control plane node -func NewCertificatesForInitialControlPlane() Certificates { - certificatesDir := DefaultCertificatesDir - - certificates := Certificates{ - &Certificate{ - Purpose: ClusterCA, - CertFile: filepath.Join(certificatesDir, "server-ca.crt"), - KeyFile: filepath.Join(certificatesDir, "server-ca.key"), - }, - &Certificate{ - Purpose: ClientClusterCA, - CertFile: filepath.Join(certificatesDir, "client-ca.crt"), - KeyFile: filepath.Join(certificatesDir, "client-ca.key"), - }, - } - - return certificates -} - -// GetByPurpose returns a certificate by the given name. -// This could be removed if we use a map instead of a slice to hold certificates, however other code becomes more complex. -func (c Certificates) GetByPurpose(purpose Purpose) *Certificate { - for _, certificate := range c { - if certificate.Purpose == purpose { - return certificate - } - } - return nil -} - -// Lookup looks up each certificate from secrets and populates the certificate with the secret data. -func (c Certificates) Lookup(ctx context.Context, ctrlclient client.Client, clusterName client.ObjectKey) error { - // Look up each certificate as a secret and populate the certificate/key - for _, certificate := range c { - s := &corev1.Secret{} - key := client.ObjectKey{ - Name: Name(clusterName.Name, certificate.Purpose), - Namespace: clusterName.Namespace, - } - if err := ctrlclient.Get(ctx, key, s); err != nil { - if apierrors.IsNotFound(err) { - if certificate.External { - return errors.WithMessage(err, "external certificate not found") - } - continue - } - return errors.WithStack(err) - } - // If a user has a badly formatted secret it will prevent the cluster from working. - kp, err := secretToKeyPair(s) - if err != nil { - return err - } - certificate.KeyPair = kp - } - return nil -} - -// EnsureAllExist ensure that there is some data present for every certificate -func (c Certificates) EnsureAllExist() error { - for _, certificate := range c { - if certificate.KeyPair == nil { - return ErrMissingCertificate - } - if len(certificate.KeyPair.Cert) == 0 { - return errors.Wrapf(ErrMissingCrt, "for certificate: %s", certificate.Purpose) - } - if !certificate.External { - if len(certificate.KeyPair.Key) == 0 { - return errors.Wrapf(ErrMissingKey, "for certificate: %s", certificate.Purpose) - } - } - } - return nil -} - -// Generate will generate any certificates that do not have KeyPair data. -func (c Certificates) Generate() error { - for _, certificate := range c { - if certificate.KeyPair == nil { - err := certificate.Generate() - if err != nil { - return err - } - } - } - return nil -} - -// SaveGenerated will save any certificates that have been generated as Kubernetes secrets. -func (c Certificates) SaveGenerated(ctx context.Context, ctrlclient client.Client, clusterName client.ObjectKey, owner metav1.OwnerReference) error { - for _, certificate := range c { - if !certificate.Generated { - continue - } - s := certificate.AsSecret(clusterName, owner) - if err := ctrlclient.Create(ctx, s); err != nil { - return errors.WithStack(err) - } - } - return nil -} - -// LookupOrGenerate is a convenience function that wraps cluster bootstrap certificate behavior. -func (c Certificates) LookupOrGenerate(ctx context.Context, ctrlclient client.Client, clusterName client.ObjectKey, owner metav1.OwnerReference) error { - // Find the certificates that exist - if err := c.Lookup(ctx, ctrlclient, clusterName); err != nil { - return err - } - - // Generate the certificates that don't exist - if err := c.Generate(); err != nil { - return err - } - - // Save any certificates that have been generated - if err := c.SaveGenerated(ctx, ctrlclient, clusterName, owner); err != nil { - return err - } - - return nil -} - -// Certificate represents a single certificate CA. -type Certificate struct { - Generated bool - External bool - Purpose Purpose - KeyPair *certs.KeyPair - CertFile, KeyFile string -} - -// Hashes hashes all the certificates stored in a CA certificate. -func (c *Certificate) Hashes() ([]string, error) { - certificates, err := cert.ParseCertsPEM(c.KeyPair.Cert) - if err != nil { - return nil, errors.Wrapf(err, "unable to parse %s certificate", c.Purpose) - } - out := make([]string, 0) - for _, c := range certificates { - out = append(out, hashCert(c)) - } - return out, nil -} - -// hashCert calculates the sha256 of certificate. -func hashCert(certificate *x509.Certificate) string { - spkiHash := sha256.Sum256(certificate.RawSubjectPublicKeyInfo) - return "sha256:" + strings.ToLower(hex.EncodeToString(spkiHash[:])) -} - -// AsSecret converts a single certificate into a Kubernetes secret. -func (c *Certificate) AsSecret(clusterName client.ObjectKey, owner metav1.OwnerReference) *corev1.Secret { - s := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: clusterName.Namespace, - Name: Name(clusterName.Name, c.Purpose), - Labels: map[string]string{ - clusterv1.ClusterLabelName: clusterName.Name, - }, - }, - Data: map[string][]byte{ - TLSKeyDataName: c.KeyPair.Key, - TLSCrtDataName: c.KeyPair.Cert, - }, - Type: clusterv1.ClusterSecretType, - } - - if c.Generated { - s.OwnerReferences = []metav1.OwnerReference{owner} - } - return s -} - -// AsFiles converts the certificate to a slice of Files that may have 0, 1 or 2 Files. -func (c *Certificate) AsFiles() []bootstrapv1.File { - out := make([]bootstrapv1.File, 0) - if len(c.KeyPair.Cert) > 0 { - out = append(out, bootstrapv1.File{ - Path: c.CertFile, - Owner: rootOwnerValue, - Permissions: "0640", - Content: string(c.KeyPair.Cert), - }) - } - if len(c.KeyPair.Key) > 0 { - out = append(out, bootstrapv1.File{ - Path: c.KeyFile, - Owner: rootOwnerValue, - Permissions: "0600", - Content: string(c.KeyPair.Key), - }) - } - return out -} - -// Generate will generate a new certificate. -func (c *Certificate) Generate() error { - // Do not generate the APIServerEtcdClient key pair. It is user supplied - if c.Purpose == APIServerEtcdClient { - return nil - } - - generator := generateCACert - if c.Purpose == ServiceAccount { - generator = generateServiceAccountKeys - } - - kp, err := generator() - if err != nil { - return err - } - c.KeyPair = kp - c.Generated = true - - return nil -} - -// AsFiles converts a slice of certificates into bootstrap files. -func (c Certificates) AsFiles() []bootstrapv1.File { - clusterCA := c.GetByPurpose(ClusterCA) - clientClusterCA := c.GetByPurpose(ClientClusterCA) - - etcdCA := c.GetByPurpose(EtcdCA) - - certFiles := make([]bootstrapv1.File, 0) - if clusterCA != nil { - certFiles = append(certFiles, clusterCA.AsFiles()...) - } - if clientClusterCA != nil { - certFiles = append(certFiles, clientClusterCA.AsFiles()...) - } - if etcdCA != nil { - certFiles = append(certFiles, etcdCA.AsFiles()...) - } - - // these will only exist if external etcd was defined and supplied by the user - apiserverEtcdClientCert := c.GetByPurpose(APIServerEtcdClient) - if apiserverEtcdClientCert != nil { - certFiles = append(certFiles, apiserverEtcdClientCert.AsFiles()...) - } - - return certFiles -} - -func secretToKeyPair(s *corev1.Secret) (*certs.KeyPair, error) { - c, exists := s.Data[TLSCrtDataName] - if !exists { - return nil, errors.Errorf("missing data for key %s", TLSCrtDataName) - } - - // In some cases (external etcd) it's ok if the etcd.key does not exist. - // TODO: some other function should ensure that the certificates we need exist. - key, exists := s.Data[TLSKeyDataName] - if !exists { - key = []byte("") - } - - return &certs.KeyPair{ - Cert: c, - Key: key, - }, nil -} - -func generateCACert() (*certs.KeyPair, error) { - x509Cert, privKey, err := newCertificateAuthority() - if err != nil { - return nil, err - } - return &certs.KeyPair{ - Cert: certs.EncodeCertPEM(x509Cert), - Key: certs.EncodePrivateKeyPEM(privKey), - }, nil -} - -func generateServiceAccountKeys() (*certs.KeyPair, error) { - saCreds, err := certs.NewPrivateKey() - if err != nil { - return nil, err - } - saPub, err := certs.EncodePublicKeyPEM(&saCreds.PublicKey) - if err != nil { - return nil, err - } - return &certs.KeyPair{ - Cert: saPub, - Key: certs.EncodePrivateKeyPEM(saCreds), - }, nil -} - -// newCertificateAuthority creates new certificate and private key for the certificate authority -func newCertificateAuthority() (*x509.Certificate, *rsa.PrivateKey, error) { - key, err := certs.NewPrivateKey() - if err != nil { - return nil, nil, err - } - - c, err := newSelfSignedCACert(key) - if err != nil { - return nil, nil, err - } - - return c, key, nil -} - -// newSelfSignedCACert creates a CA certificate. -func newSelfSignedCACert(key *rsa.PrivateKey) (*x509.Certificate, error) { - cfg := certs.Config{ - CommonName: "kubernetes", - } - - now := time.Now().UTC() - - tmpl := x509.Certificate{ - SerialNumber: new(big.Int).SetInt64(0), - Subject: pkix.Name{ - CommonName: cfg.CommonName, - Organization: cfg.Organization, - }, - NotBefore: now.Add(time.Minute * -5), - NotAfter: now.Add(time.Hour * 24 * 365 * 10), // 10 years - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, - MaxPathLenZero: true, - BasicConstraintsValid: true, - MaxPathLen: 0, - IsCA: true, - } - - b, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, key.Public(), key) - if err != nil { - return nil, errors.Wrapf(err, "failed to create self signed CA certificate: %+v", tmpl) - } - - c, err := x509.ParseCertificate(b) - return c, errors.WithStack(err) -} diff --git a/util/secret/consts.go b/util/secret/consts.go deleted file mode 100644 index f207f47e..00000000 --- a/util/secret/consts.go +++ /dev/null @@ -1,57 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package secret - -// Purpose is the name to append to the secret generated for a cluster. -type Purpose string - -const ( - // KubeconfigDataName is the key used to store a Kubeconfig in the secret's data field. - KubeconfigDataName = "value" - - // TLSKeyDataName is the key used to store a TLS private key in the secret's data field. - TLSKeyDataName = "tls.key" - - // TLSCrtDataName is the key used to store a TLS certificate in the secret's data field. - TLSCrtDataName = "tls.crt" - - // Kubeconfig is the secret name suffix storing the Cluster Kubeconfig. - Kubeconfig = Purpose("kubeconfig") - - // ClusterCA is the secret name suffix for APIServer CA. - ClusterCA = Purpose("ca") - - // ClientClusterCA is the secret name suffix for APIServer CA. - ClientClusterCA = Purpose("cca") - - // EtcdCA is the secret name suffix for the Etcd CA - EtcdCA Purpose = "etcd" - - // ServiceAccount is the secret name suffix for the Service Account keys - ServiceAccount Purpose = "sa" - - // FrontProxyCA is the secret name suffix for Front Proxy CA - FrontProxyCA Purpose = "proxy" - - // APIServerEtcdClient is the secret name of user-supplied secret containing the apiserver-etcd-client key/cert - APIServerEtcdClient Purpose = "apiserver-etcd-client" -) - -var ( - // allSecretPurposes defines a lists with all the secret suffix used by Cluster API - allSecretPurposes = []Purpose{Kubeconfig, ClusterCA, EtcdCA, ServiceAccount, FrontProxyCA, APIServerEtcdClient} -) diff --git a/util/secret/doc.go b/util/secret/doc.go deleted file mode 100644 index bfe0749e..00000000 --- a/util/secret/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package secret provides a secret store for storing secrets. -package secret diff --git a/util/secret/secret.go b/util/secret/secret.go deleted file mode 100644 index de88b144..00000000 --- a/util/secret/secret.go +++ /dev/null @@ -1,71 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package secret - -import ( - "context" - "fmt" - "strings" - - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// Get retrieves the specified Secret (if any) from the given -// cluster name and namespace. -func Get(ctx context.Context, c client.Reader, cluster client.ObjectKey, purpose Purpose) (*corev1.Secret, error) { - return GetFromNamespacedName(ctx, c, cluster, purpose) -} - -// GetFromNamespacedName retrieves the specified Secret (if any) from the given -// cluster name and namespace. -func GetFromNamespacedName(ctx context.Context, c client.Reader, clusterName client.ObjectKey, purpose Purpose) (*corev1.Secret, error) { - secret := &corev1.Secret{} - secretKey := client.ObjectKey{ - Namespace: clusterName.Namespace, - Name: Name(clusterName.Name, purpose), - } - - if err := c.Get(ctx, secretKey, secret); err != nil { - return nil, err - } - - return secret, nil -} - -// Name returns the name of the secret for a cluster. -func Name(cluster string, suffix Purpose) string { - return fmt.Sprintf("%s-%s", cluster, suffix) -} - -// ParseSecretName return the cluster name and the suffix Purpose in name is a valid cluster secrets, -// otherwise it return error. -func ParseSecretName(name string) (string, Purpose, error) { - separatorPos := strings.LastIndex(name, "-") - if separatorPos == -1 { - return "", "", errors.Errorf("%q is not a valid cluster secret name. The purpose suffix is missing", name) - } - clusterName := name[:separatorPos] - purposeSuffix := Purpose(name[separatorPos+1:]) - for _, purpose := range allSecretPurposes { - if purpose == purposeSuffix { - return clusterName, purposeSuffix, nil - } - } - return "", "", errors.Errorf("%q is not a valid cluster secret name. Invalid purpose suffix", name) -} diff --git a/util/util.go b/util/util.go deleted file mode 100644 index 10ea671f..00000000 --- a/util/util.go +++ /dev/null @@ -1,84 +0,0 @@ -/* - Copyright 2022 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// Package util contains utility functions -package util - -import ( - "context" - - "github.com/go-logr/logr" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/controller-runtime/pkg/client" - - infrav1 "github.com/kubesphere/kubekey/v3/api/v1beta1" - "github.com/kubesphere/kubekey/v3/pkg/scope" -) - -// GetInfraCluster returns the infrastructure cluster object corresponding to a Cluster. -func GetInfraCluster(ctx context.Context, c client.Client, log logr.Logger, cluster *clusterv1.Cluster, controllerName string, - dataDir string) (*scope.ClusterScope, error) { - kkCluster := &infrav1.KKCluster{} - infraClusterName := client.ObjectKey{ - Namespace: cluster.Spec.InfrastructureRef.Namespace, - Name: cluster.Spec.InfrastructureRef.Name, - } - - if err := c.Get(ctx, infraClusterName, kkCluster); err != nil { - return nil, err - } - - // Create the cluster scope - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ - Client: c, - Logger: &log, - Cluster: cluster, - KKCluster: kkCluster, - ControllerName: controllerName, - RootFsBasePath: dataDir, - }) - if err != nil { - return nil, err - } - - return clusterScope, nil -} - -// GetOwnerKKMachine returns the Machine object owning the given object. -func GetOwnerKKMachine(ctx context.Context, c client.Client, obj metav1.ObjectMeta) (*infrav1.KKMachine, error) { - for _, ref := range obj.OwnerReferences { - gv, err := schema.ParseGroupVersion(ref.APIVersion) - if err != nil { - return nil, err - } - if ref.Kind == "KKMachine" && gv.Group == infrav1.GroupVersion.Group { - return GetKKMachineByName(ctx, c, obj.Namespace, ref.Name) - } - } - return nil, nil -} - -// GetKKMachineByName finds and return a Machine object using the specified params. -func GetKKMachineByName(ctx context.Context, c client.Client, namespace, name string) (*infrav1.KKMachine, error) { - m := &infrav1.KKMachine{} - key := client.ObjectKey{Name: name, Namespace: namespace} - if err := c.Get(ctx, key, m); err != nil { - return nil, err - } - return m, nil -} diff --git a/version/components.json b/version/components.json deleted file mode 100644 index f3f48a45..00000000 --- a/version/components.json +++ /dev/null @@ -1,1219 +0,0 @@ -{ - "kubeadm": { - "amd64": { - "v1.19.0": "88ce7dc5302d8847f6e679aab9e4fa642a819e8a33d70731fb7bc8e110d8659f", - "v1.19.8": "9c6646cdf03efc3194afc178647205195da4a43f58d0b70954953f566fa15c76", - "v1.19.9": "917712bbd38b625aca456ffa78bf134d64f0efb186cc5772c9844ba6d74fd920", - "v1.19.15": "7fd8558856497d563a4e9ee0235183d2887751e68b11296b1064c32273933cf7", - "v1.20.4": "dcc5629da2c31a000b9b50db077b1cd51a6840e08233fd64b67e37f3f098c392", - "v1.20.6": "ff6fca46edeccd8a4dbf162079d0b3d27841b04885b3f47f80377b3a93ab1533", - "v1.20.10": "da5864968a38e0bf2317965e87b5425e1b9101a49dd5178f2e967c0a46547270", - "v1.21.0": "7bdaf0d58f0d286538376bc40b50d7e3ab60a3fe7a0709194f53f1605129550f", - "v1.21.1": "1553c07a6a777c4cf71d45d5892915f0ea6586b8a80f9fea39e7a659d6315d42", - "v1.21.2": "6a83e52e51f41d67658a13ce8ac9deb77a6d82a71ced2d106756f6d38756ec00", - "v1.21.3": "82fff4fc0cdb1110150596ab14a3ddcd3dbe53f40c404917d2e9703f8f04787a", - "v1.21.4": "286794aed41148e82a77087d79111052ea894796c6ae81fc463275dcd848f98d", - "v1.21.5": "e384171fcb3c0de924904007bfd7babb0f970997b93223ed7ffee14d29019353", - "v1.21.6": "fef4b40acd982da99294be07932eabedd476113ce5dc38bb9149522e32dada6d", - "v1.21.7": "c4480121b629a0f563f718aa11440ae26a569e37e0229c093a5785c90725a03c", - "v1.21.8": "51d266e91e2aec0e994c046b4d80901a1b1e7be05e30b83461f0563571f1224d", - "v1.21.9": "3333116f9f0d72e0598f52dcbef7ecab1ce88192fdcfd5384ca919fdc075e8d5", - "v1.21.10": "61aaadd98806d979b65e031a144d9379390d26ccb5383d47bdd8b7c727e94a7b", - "v1.21.11": "3514ea5acaae9c2779a341deb24832df17722cb612fa7a78d34f602f91e94d17", - "v1.21.12": "f6ef1d2d19ba0aaaba4c57c4eda94e2725c3f7e9412feb5d6fe12c1827e7c1cb", - "v1.21.13": "5d25cc16bd38e0aaf7010d115827f7d95a1dcf7343e6b096b3df1b700ce23f7e", - "v1.21.14": "3b1da35298d062540f9ecad0f18cf4e44be0c7d37c5e430ed0cb56d6c0fe5ebc", - "v1.22.0": "90a48b92a57ff6aef63ff409e2feda0713ca926b2cd243fe7e88a84c483456cc", - "v1.22.1": "50a5f0d186d7aefae309539e9cc7d530ef1a9b45ce690801655c2bee722d978c", - "v1.22.2": "4ff09d3cd2118ee2670bc96ed034620a9a1ea6a69ef38804363d4710a2f90d8c", - "v1.22.3": "3964e6fd46052eb4a9672421d8e8ce133b83b45abb77481b688dc6375390e480", - "v1.22.4": "33b799df2941f12a53ffe995d86a385c35d3c543f9d2c00c0cdb47ec91a98c5c", - "v1.22.5": "a512be0fa429f43d3457472efd73529cd2ba2cd54ef714faf6b69486beea054f", - "v1.22.6": "0bf8e47ad91215cd8c5e0ded565645aeb1ad6f0a9223a2486eb913bff929d472", - "v1.22.7": "7e4be37fc5ddeeae732886bf83c374198813e76d84ed2f6590145e08ece1a8b2", - "v1.22.8": "fc10b4e5b66c9bfa6dc297bbb4a93f58051a6069c969905ef23c19680d8d49dc", - "v1.22.9": "e3061f3a9c52bff82ae740c928fe389a256964a5756d691758bf3611904d7183", - "v1.22.10": "df5e090a3c0e24b92b26f22f1d7689b6ea860099ea89b97edf5d4c19fa6da0ca", - "v1.22.11": "da3594b4e905627fd5c158531280e40a71dadf44f1f0b6c061a1b729a898dd9b", - "v1.22.12": "9410dcff069993caa7dfe783d35ac2d929ec258a2c3a4f0c3f269f1091931263", - "v1.22.13": "acbb0dd67b7656d0c70049484ba31c1981b803be0ae8f430dacad67e3e06c121", - "v1.22.14": "c8343a3e8a3056d922e466733486ccbbd8efd01a453a9e93e1cf8a164281e6b2", - "v1.22.15": "c84799162c33f758facbe9d6cbabfbda1ca3f74e87386e98af8711278c706872", - "v1.22.16": "0b9c7b88d63a82a31e697a1a90b42f36a49d6056f567adc793013fee1ffe709d", - "v1.22.17": "75244faf5726baf432ff2a76d5f188772173adb5ca1c33634d56ba13dbd6e4dc", - "v1.23.0": "e21269a058d4ad421cf5818d4c7825991b8ba51cd06286932a33b21293b071b0", - "v1.23.1": "4d5766cb90050ee84e15df5e09148072da2829492fdb324521c4fa6d74d3aa34", - "v1.23.2": "58487391ec37489bb32fe532e367995e9ecaeafdb65c2113ff3675e7a8407219", - "v1.23.3": "57ec7f2921568dcf4cda0699b877cc830d49ddd2709e035c339a5afc3b83586f", - "v1.23.4": "c91912c9fd34a50492f889e08ff94c447fdceff150b588016fecc9051a1e56b8", - "v1.23.5": "8eebded187ee84c97003074eaa347e34131fef3acdf3e589a9b0200f94687667", - "v1.23.6": "9213c7d738e86c9a562874021df832735236fcfd5599fd4474bab3283d34bfd7", - "v1.23.7": "d7d863213eeb4791cdbd7c5fd398cf0cc2ef1547b3a74de8285786040f75efd2", - "v1.23.8": "edbd60fd6a7e11c71f848b3a6e5d1b5a2bb8ebd703e5490caa8db267361a7b89", - "v1.23.9": "947571c50ab840796fdd4ffb129154c005dfcb0fe83c6eff392d46cf187fd296", - "v1.23.10": "43d186c3c58e3f8858c6a22bc71b5441282ac0ccbff6f1d0c2a66ee045986b64", - "v1.23.11": "2f10bd298a694d3133ea19192b796a106c282441e4148c114c39376042097692", - "v1.23.12": "bf45d00062688d21ff479bf126e1259d0ce3dee1c5c2fcd803f57497cd5e9e83", - "v1.23.13": "ff86af2b5fa979234dd3f9e7b04ec7d3017239a58417397153726d8077c4ac89", - "v1.23.14": "46c847e2699839b9ccf6673f0b946c4778a3a2e8e463d15854ba30d3f0cbd87a", - "v1.23.15": "63329e21be8367628f71978cfc140c74ce9cb0336abd9c4802ca7d20d5dec3c3", - "v1.23.16": "511b8d915b8e7ab3c0e6aef905c7ac7901c39d2c67fbc099dda28b498a9871d8", - "v1.23.17": "6d3f732fe1eabd91c98ff0ee66c6c8b4fcbdee9e99c2c8606f0fa5ff57b4ea65", - "v1.24.0": "5e58a29eaaf69ea80e90d9780d2a2d5f189fd74f94ec3bec9e3823d472277318", - "v1.24.1": "15e3193eecbc69330ada3f340c5a47999959bc227c735fa95e4aa79470c085d0", - "v1.24.2": "028f73b8e7c2ae389817d34e0cb829a814ce2fac0a535a3aa0708f3133e3e712", - "v1.24.3": "406d5a80712c45d21cdbcc51aab298f0a43170df9477259443d48eac116998ff", - "v1.24.4": "9ec08e0905c0a29a68676ba9f6dd7de73bef13cfa2b846a45e1c2189572dc57c", - "v1.24.5": "3b9c1844ec0fc3c94015d63470b073a7b219082b6a6424c6b0da9cf97e234aeb", - "v1.24.6": "7f4443fd42e0e03f6fd0c7218ca7e2634c9255d5f9d7c581fe362e19098aec4c", - "v1.24.7": "8b67319d28bf37e8e7c224954dc778cbe946f2bb0ed86975d8caa83d51c955ee", - "v1.24.8": "9fea42b4fb5eb2da638d20710ebb791dde221e6477793d3de70134ac058c4cc7", - "v1.24.9": "20406971ae71886f7f8ee7b9a33c885391ae64da561fb679d5819f2ccc19ac9f", - "v1.24.10": "5e29917dc277a8bc4b90bf9dbed8d3dca903fd7cbf7f12c2e256fe22e9f2a1f9", - "v1.24.11": "9123cdda3118772794198f784e4f39f34212ebaa919bdddaf611190d7c4abf38", - "v1.24.12": "a6daad39597a9d3d4c49a44ce2b77bb45290855085cbfe2e1b20afd84f40d143", - "v1.24.13": "91a65b7415a1a7e3dbdb23683eff9e2ace5a725a1bcd52b10695dd49adb6787e", - "v1.24.14": "5de81e7fa23e513411c43ca2e59fe6c64848c7a7108472a7a7931f867e28d223", - "v1.24.15": "0f4838139affb7b1a131d5d0a0b93d4ada0d1d22da5d04d5d3599018adc130b7", - "v1.24.16": "4c385d35a0e1506e95adb6230d6d2c14e61fc8a99e4746ca1c2d687bfd963a53", - "v1.24.17": "774bae9f86c093fd28c7649cf4a84cb3555a70d29ea5dba438abc990dd0febc9", - "v1.25.0": "10b30b87af2cdc865983d742891eba467d038f94f3926bf5d0174f1abf6628f8", - "v1.25.1": "adaa1e65c1cf9267a01e889d4c13884f883cf27948f00abb823f10486f1a8420", - "v1.25.2": "63ee3de0c386c6f3c155874b46b07707cc72ce5b9e23f336befd0b829c1bd2ad", - "v1.25.3": "01b59ce429263c62b85d2db18f0ccdef076b866962ed63971ff2bd2864deea7b", - "v1.25.4": "b8a6119d2a3a7c6add43dcf8f920436bf7fe71a77a086e96e40aa9d6f70be826", - "v1.25.5": "af0b25c7a995c2d208ef0b9d24b70fe6f390ebb1e3987f4e0f548854ba9a3b87", - "v1.25.6": "d8bf16d1a808dce10d4eb9b391ddd6ee8a81e94c669441f20b1227083dbc4417", - "v1.25.7": "54e369043d5c7ac320ccbd51757019274dbfefce36c9abee746e387ac8203704", - "v1.25.8": "2ae844776ac48273d868f92a7ed9d54b4a6e9b0e4d05874d77b7c0f4bfa60379", - "v1.25.9": "157be24d998111a51d52db016f9010cd6418506a028f87b5a712f518b392a3f3", - "v1.25.10": "7300211efa962d1ca27121ae68be6f06c7f2dca4ca8e5087a2a69f36daa6b9dc", - "v1.25.11": "6ff43cc8266a21c7b62878a0a9507b085bbb079a37b095fab5bcd31f2dbd80e0", - "v1.25.12": "293252f0a1727bfad4ef4fe99d704a56ecea45e39b0ea77f629c55da39e377da", - "v1.25.13": "4694df9c5d700280c186980907ec8e695364f461b20e868336a71edabac2253e", - "v1.25.14": "6cce9224e8b939bb0c218ab1b047a934a8c2b23f07c7ade4586b5e1a4013c80f", - "v1.25.15": "f79ab4d8f3a11c9f6f1b2c040552d4e1b0462fa9f9ddde7431b990e4b6e387ff", - "v1.25.16": "11c70502ac5bad303b5b4103b9eb5b2a83376cf6a1bce878b6018c6ca44a7d6e", - "v1.26.0": "72631449f26b7203701a1b99f6914f31859583a0e247c3ac0f6aaf59ca80af19", - "v1.26.1": "1531abfe96e2e9d8af9219192c65d04df8507a46a081ae1e101478e95d2b63da", - "v1.26.2": "277d880dc6d79994fd333e49d42943b7c9183b1c4ffdbf9da59f806acec7fd82", - "v1.26.3": "87a1bf6603e252a8fa46be44382ea218cb8e4f066874d149dc589d0f3a405fed", - "v1.26.4": "aa1a137aa2c3427f199ff652c96b11d6b124358296996eb7b8cbde220607b2fe", - "v1.26.5": "793767419c382bae2dc2c9396baafbf051bfa3214accf40dcd7c5ea405583802", - "v1.26.6": "ba699c3c26aaf64ef46d34621de9f3b62e37656943e09f23dc3bf5aa7b3f5094", - "v1.26.7": "812e6d0e94a3fc77d3e9d09dbe709190b77408936cc4e960d916e8401be11090", - "v1.26.8": "233a89277ca49dbd666b7391c6c0e43c33d2f08052d5b93e9cd0100ee69430c8", - "v1.26.9": "73e128821dd1f799a75c922218d12f6c4618b8e29cc7dae2a7390fb80092d3d9", - "v1.26.10": "27ed1d857f4a315f3d059168c6e25fdbf0559f9c8e59bab6c50e7921f74dadbf", - "v1.26.11": "58f886e39e517ba1a92493f136e80f1b6ea9362966ad9d2accdf2133004161f2", - "v1.26.12": "5a5d65acefb50010859be8ffba8e6e059d552ae357e3101c12c62e747a9416a2", - "v1.27.0": "78d0e04705a7bdb76a514d60f60c073b16334b15f57ee87f064354ca8a233e80", - "v1.27.1": "c7d32d698e99b90f877025104cb4a9f3f8c707e99e6817940f260135b6d1ad0a", - "v1.27.2": "95c4bfb7929900506a42de4d92280f06efe6b47e0a32cbc1f5a1ed737592977a", - "v1.27.3": "2cd663f25c2490bd614a6c0ad9089a47ef315caf0dbdf78efd787d5653b1c6e3", - "v1.27.4": "7be21d6fb3707fbbe8f0db0403db6234c8af773b941f931bf8248759ee988bcd", - "v1.27.5": "35df8efa6e1bc864ed3c48a665caed634a5c46cfd7f41cda5ad66defdfddb2aa", - "v1.27.6": "2bcdd68957ec25d0689bb56f32b4ec86e38463d2691d5ea21cd109c7afa3aa7c", - "v1.27.7": "bc589219a003b3b94c114e4bcf20549a02657a0c6e5c73f588b37817148892d2", - "v1.27.8": "f8864769b8b2d7a14f53eb983f23317ff14d68ab76aba71e9de17ce84c38d4eb", - "v1.27.9": "78dddac376fa2f04116022cb44ed39ccb9cb0104e05c5b21b220d5151e5c0f86", - "v1.28.0": "12ea68bfef0377ccedc1a7c98a05ea76907decbcf1e1ec858a60a7b9b73211bb", - "v1.28.1": "6134dbc92dcb83c3bae1a8030f7bb391419b5d13ea94badd3a79b7ece75b2736", - "v1.28.2": "6a4808230661c69431143db2e200ea2d021c7f1b1085e6353583075471310d00", - "v1.28.3": "ce3848b1dfa562e0fa2f911a3d8e3bb07ba040eea76654d68e213315c8846ac0", - "v1.28.4": "b4d2531b7cddf782f59555436bc098485b5fa6c05afccdeecf0d62d21d84f5bd", - "v1.28.5": "2b54078c5ea9e85b27f162f508e0bf834a2753e52a57e896812ec3dca92fe9cd", - "v1.29.0": "629d4630657caace9c819fd3797f4a70c397fbd41a2a7e464a0507dad675d52c" - }, - "arm64": { - "v1.19.0": "db1c432646e6e6484989b6f7191f3610996ac593409f12574290bfc008ea11f5", - "v1.19.8": "dfb838ffb88d79e4d881326f611ae5e5999accb54cdd666c75664da264b5d58e", - "v1.19.9": "403c767bef0d681aebc45d5643787fc8c0b9344866cbd339368637a05ea1d11c", - "v1.19.15": "44bbd07a5fc048bccfe4c0e7efd8abb9bd8daa420898bbe8519da91cebadf483", - "v1.20.4": "c3ff7f944826889a23a002c85e8f9f9d9a8bc95e9083fbdda59831e3e34245a7", - "v1.20.6": "33837e290bd76fcb16af27db0e814ec023c25e6c41f25a0907b48756d4a2ffc2", - "v1.20.10": "ec1f8df0f57b8aa6bddce2d6bb8d0503e016b022ba8a5f113ddf412d9a99c03c", - "v1.21.0": "50bb95d1827455346b5643dcf83a52520733c3a582b8b1ffb50f04a8e66f00e7", - "v1.21.1": "1c9a93ac74f2756c1eb40a9d18bb7e146eeab0b33177c0f66f5e617ed7261d1b", - "v1.21.2": "245125dc436f649466123a2d2c922d17f300cbc20d2b75edad5e42d734ead4a3", - "v1.21.3": "5bff1c6cd1d683ce191d271b968d7b776ae5ed7403bdab5fa88446100e74972c", - "v1.21.4": "30645f57296281d214a9dd787a90bd16207df4b1fca7ac320913c616818a92cd", - "v1.21.5": "5a273b023eaa60d7820436b0f0062c4bd467274d6f2b86a9e13270c91d663618", - "v1.21.6": "498325da2521ce67b27902967daf4087153c5797070e03bf0bdd7c846f4d61a8", - "v1.21.7": "d2d17f37f1e4de446cf75f60a2a6f7fba3cbc8e27a1d176cfa0fa48862fad4bc", - "v1.21.8": "abf2d57cb42e8dfbcb3632dd278991bcf422891cc91e3967e00f7f45183bb43e", - "v1.21.9": "8947309c985911a99fb0a6e30f9ca85d9b7adc1215149e45e5be150c7e5e5de9", - "v1.21.10": "7607bfd40317a24a276e452b46a26a7298dde2988fce826f1ee0fe9355eae786", - "v1.21.11": "97117a6d984ff88628654494181b62502cbf4c310af70d4de92dab35482900e5", - "v1.21.12": "6b59aab97cabb8becdd0aa1260bc0553998c8e6511507c07b0fa231c0865211d", - "v1.21.13": "dad351cf95224f7eea54d12c84141420a750a3f6289eb4f442b9c0488def8858", - "v1.21.14": "7f175a51f6bd84a782a5f6325c5e7e523194a31c37d606b0f1ae2ee9a2ba3e7c", - "v1.22.0": "9fc14b993de2c275b54445255d7770bd1d6cdb49f4cf9c227c5b035f658a2351", - "v1.22.1": "85df7978b2e5bb78064ed0bcce14a39d105a1a3968bb92ee5d2f96a1fa09ed12", - "v1.22.2": "77b4c6a56ae0ec142f54a6f5044a7167cdd7193612b04b77bf433ffe1d1918ef", - "v1.22.3": "dcd1ecfb7f51fb3929b9c63a984b00cf6baa6136e1d58f943ee2c9a47af5875d", - "v1.22.4": "3dfb128e108a3f07c53cae777026f529784a057628c721062d8fdd94b6870b69", - "v1.22.5": "47aa54533289277ac13419c16ffd1a2c35c7af2d6a571261e3d728990bc5fc7d", - "v1.22.6": "bc10e4fb42a182515f4232205bea53f90270b8f80ec1a6c1cc3301bff05e86b7", - "v1.22.7": "2ae0287769a70f442757e49af0ecd9ca2c6e5748e8ba72cb822d669a7aeeb8fa", - "v1.22.8": "67f09853d10434347eb75dbb9c63d57011ba3e4f7e1b320a0c30612b8185be8c", - "v1.22.9": "0168c60d1997435b006b17c95a1d42e55743048cc50ee16c8774498aa203a202", - "v1.22.10": "8ea22a05b428de70a430711e8f75553e1be2925977ab773b5be1c240bc5b9fcd", - "v1.22.11": "15e1cba65f0db4713bf45ee23dbd01dd30048d20ad97ef985d6b9197f8ae359a", - "v1.22.12": "d0469a3008411edb50f6562e00f1df28123cf2dc368f1538f1b41e27b0482b1c", - "v1.22.13": "2c42aadc99b46b6b5684acc7dfa630c67cb12c19b17df4cea3d2091ef5753011", - "v1.22.14": "cc03d2bdf29900244ff59614e007786f3ff4820e4d77709067298f5b2db20a9e", - "v1.22.15": "7f34e1e96831ae4fac769caa0dfd3646d2b02f6e0516394d814ca39d2fac4625", - "v1.22.16": "7b2fc70336487aadae13fb48a74ba41d9198d35eeef1ddda7e85615da8f798bf", - "v1.22.17": "90a1e3c98fabaf4c6dfca5cef4d33a3212a9d624ec2e6249e7ac747f10dcad67", - "v1.23.0": "989d117128dcaa923b2c7a917a03f4836c1b023fe1ee723541e0e39b068b93a6", - "v1.23.1": "eb865da197f4595dec21e6fb1fa1751ef25ac66b64fa77fd4411bbee33352a40", - "v1.23.2": "a29fcde7f92e1abfe992e99f415d3aee0fa381478b4a3987e333438b5380ddff", - "v1.23.3": "5eceefa3ca737ff1532f91bdb9ef7162882029a2a0300b4348a0980249698398", - "v1.23.4": "90fd5101e321053cdb66d165879a9cde18f19ba9bb8eae152fd4f4fcbe497be1", - "v1.23.5": "22a8468abc5d45b3415d694ad52cc8099114248c3d1fcf4297ec2b336f5cc274", - "v1.23.6": "a4db7458e224c3a2a7b468fc2704b31fec437614914b26a9e3d9efb6eecf61ee", - "v1.23.7": "65fd71aa138166039b7f4f3695308064abe7f41d2f157175e6527e60fb461eae", - "v1.23.8": "9b3d8863ea4ab0438881ccfbe285568529462bc77ef4512b515397a002d81b22", - "v1.23.9": "a0a007023db78e5f78d3d4cf3268b83f093201847c1c107ffb3dc695f988c113", - "v1.23.10": "42e957eebef78f6462644d9debc096616054ebd2832e95a176c07c28ebed645c", - "v1.23.11": "329d9aa9461baf4a7b7225e664ec1ecd61512b937e1f160f9a303bc0f0d44bbb", - "v1.23.12": "d05f6765a65f7541d07aad989ee80cd730c395f042afbe0526f667ea1a0b2947", - "v1.23.13": "462971d5822c91598754dfaa9c4c8d46a8c74aefef0f4dbbc8be31c4f0d18855", - "v1.23.14": "7c21c1fa6a852b10ddea7bd1797ce8b4498d6898014d17d20748307e510a0826", - "v1.23.15": "8bb17c69ad71bb1230dbe1e598c6ae07390b57e3ba32928f28e83742105424d0", - "v1.23.16": "09d96cb798cf2468da79d1aabdcc3b08b3516dbcd206d369e7484113241b5f3e", - "v1.23.17": "cf1bca6b464f30ea078a9cf4d902033fb80527b03c2f39409e19fb8b3886c75e", - "v1.24.0": "3e0fa21b8ebce04ca919fdfea7cc756e5f645166b95d6e4b5d9912d7721f9004", - "v1.24.1": "04f18fe097351cd16dc91cd3bde979201916686c6f4e1b87bae69ab4479fda04", - "v1.24.2": "bd823b934d1445a020f8df5fe544722175024af62adbf6eb27dc7250d5db0548", - "v1.24.3": "ea0fb451b69d78e39548698b32fb8623fad61a1a95483fe0add63e3ffb6e31b5", - "v1.24.4": "18de228f6087a2e5243bffcd2cc88c40180a4fa83e4de310ad071b4620bdd8b6", - "v1.24.5": "a68c6dd24ef47825bb34a2ad430d76e6b4d3cbe92187363676993d0538013ac2", - "v1.24.6": "211b8d1881468bb673b26036dbcfa4b12877587b0a6260ffd55fd87c2aee6e41", - "v1.24.7": "ee946d82173b63f69be9075e218250d4ab1deec39d17d600b16b6743e5dca289", - "v1.24.8": "6f35562001e859f2a76a89c0da61f09433cc6628ccbc3992e82a977e0e348870", - "v1.24.9": "57c61562a9de4cc78f276f665d7f04666607b17e3ad0fa6c14be64ad85c80951", - "v1.24.10": "dfa768a684ee9854da4aee0a39164c0ce2e07156a0f0d284d9153c5eb14584b2", - "v1.24.11": "3986f6a48af8f6d2c1cdba1e60e347077e6d1882df984f422b7d1d77eb82005e", - "v1.24.12": "e3f7864b3b98c8cf1a3284911b20393f4b1fc5f9a4524ae7233258a7f9fbb1e9", - "v1.24.13": "c3b8d75bd6bed4d34786ba95d31198a967059166b1c929222a7bef238b08d829", - "v1.24.14": "953ede1b5aad5433545b715777da9aacc88318fbda34e0e0027ee5d7fb6b7c67", - "v1.24.15": "a3dc47ef4691f53d69ca8c7747e7668a4b620105ec6856f326badbe1d3d86c2d", - "v1.24.16": "6d52be09c9075895bacb8e65caec7147340809afc3a70afb11f27aa47579b427", - "v1.24.17": "1d484faddd1b789c67cdadb315d53250d9906e5b727c9254131c01b2f8f17beb", - "v1.25.0": "07d9c6ffd3676502acd323c0ca92f44328a1f0e89a7d42a664099fd3016cf16b", - "v1.25.1": "f4d57d89c53b7fb3fe347c9272ed40ec55eab120f4f09cd6b684e97cb9cbf1f0", - "v1.25.2": "437dc97b0ca25b3fa8d74b39e4059a77397b55c1a6d16bddfd5a889d91490ce0", - "v1.25.3": "61bb61eceff78b44be62a12bce7c62fb232ce1338928e4207deeb144f82f1d06", - "v1.25.4": "3f5b273e8852d13fa39892a30cf64928465c32d0eb741118ba89714b51f03cd5", - "v1.25.5": "426dddad1c60b7617f4095507cef524d76ec268a0201c1df154c108287a0b98e", - "v1.25.6": "54256821830d23b458a29f5d4695331d600e57644604ef68d0e35e5d2a4ffb4b", - "v1.25.7": "64bd532bed1814a28d021e227d18d81cf5b93ac05763a5a5fa6a38b6bb55caee", - "v1.25.8": "e7f0c738e48d905eae145631497a9ef59e792300e5247be2a1fbaa0a8907b308", - "v1.25.9": "cd15d440685f54a2c7a6cf1d96093fa7a55b9dfae8c48448b3ea92a922c07c72", - "v1.25.10": "373766681e012768d7302ce571f06c82ea18636ae95fe0b6f0fa474e51bf4deb", - "v1.25.11": "570d87d56a24778bd0854270eeddc8bcfb275f1c711cced5b5948f631e0c3ede", - "v1.25.12": "6a22e2e830f9df16a96a1ac5a4034b950b89a0cc90b19dc1fb104b268e4cd251", - "v1.25.13": "d5380bd3f0562aee30d888f22b5650c7af54da83d9fe5187821bcedf21885a11", - "v1.25.14": "525181225d963ddbc17765587a7b5919aa68d7264a197f6a1359f32e7f4a2e03", - "v1.25.15": "e3f152ae529dd6363b2748f39d219f87490f3e995e8bd5332e756c4df692f5f4", - "v1.25.16": "55cc8e3c5985858b9f683bf6c7352d76f073d3dc136f450e8761c0ed7092c0f3", - "v1.26.0": "652844c9518786273e094825b74a1988c871552dc6ccf71366558e67409859d1", - "v1.26.1": "db101c4bb8e33bd69241de227ed317feee6d44dbd674891e1b9e11c6e8b369bb", - "v1.26.2": "f210d8617acf7c601196294f7ca97e4330b75dad00df6b8dd12393730c501473", - "v1.26.3": "e9a7dbca77f9576a98af1db8747e9dc13e930e40295eaa259dd99fd6e17a173f", - "v1.26.4": "a97052d393e60027c354e97c88493aa14a76c8cfb7418bbdf8425b3711d86e3a", - "v1.26.5": "d7eede9b44850e16cbe4bb8946a79c03c2c0272f7adc726e63b3a1ac09f13b55", - "v1.26.6": "003c7740750ad92d2ff3d58d4a15015906c120c93c7aa605ba98edd936061542", - "v1.26.7": "34192ceac2287029b36e2d6b682e55dee245ae622701dc3b36bd3203019b18d1", - "v1.26.8": "f12d5d748abb8586723b78a2f0300f88faf0391f56d4d49f1ad1cef74160a1b5", - "v1.26.9": "14c87cbb9a3fa02308a9546aad192ce2d93e5d1d0296d28ba449079e6a1cb2b2", - "v1.26.10": "1ddcb47ee4f7171736dbacc046a7ceae55411ee09920435c3821b530f4650428", - "v1.26.11": "a13318c1493e58a9f7c4359c79443f3c86a690ec601bcc76308c809d8d61edb8", - "v1.26.12": "2dee03d460e8a1b3f30e5ef48ce94f5eeb4ff4550e65860e6e2d94b368c2cde6", - "v1.27.0": "acd805c6783b678ee0068b9dd8165bbfd879c345fd9c25d6a978dbc965f48544", - "v1.27.1": "024a59cd6fc76784b597c0c1cf300526e856e8c9fefa5fa7948158929b739551", - "v1.27.2": "8f01f363f7c7f92de2f2276124a895503cdc5a60ff549440170880f296b087eb", - "v1.27.3": "495e2193ed779d25584b4b532796c2270df0f7139ef15fb89dc7980603615ef4", - "v1.27.4": "b4ede8a18ef3d1cfa61e6fbca8fcab02f8eee3d0770d2329490fa7be90a4cae4", - "v1.27.5": "3023ef1d2eff885af860e13c8b9fcdb857d259728f16bf992d59c2be522cec82", - "v1.27.6": "faec35315203913b835e9b789d89001a05e072943c960bcf4de1e331d08e10c8", - "v1.27.7": "46d7c43532233906919a53ee0e03ab04ab9e08514392d17a86f058e0364cda4b", - "v1.27.8": "0d0f5b2781d663d314e785d14361aa5a09cfaf6e1694aa3cc731f4f06342ec13", - "v1.27.9": "d3d022842b0b8e4661222e8873249f5acafdbef52fd1bfb98152a582352b3c40", - "v1.28.0": "b9b473d2d9136559b19eb465006af77df45c09862cd7ce6673a33aae517ff5ab", - "v1.28.1": "7d2f68917470a5d66bd2a7d62897f59cb4afaeffb2f26c028afa119acd8c3fc8", - "v1.28.2": "010789a94cf512d918ec4a3ef8ec734dea0061d89a8293059ef9101ca1bf6bff", - "v1.28.3": "dcb37d78ccdfe9d8dd6f100e188ddc6e3f5570d0c49db68470073683b453a1e7", - "v1.28.4": "a4422780020954436b8e76ab1c59b68c5581a54432dd3e566c4709bb40c8d4f9", - "v1.28.5": "22bb6b3377204e93d008f33ac4924d77adca1478f1ae3b515c03476ba54f1adc", - "v1.29.0": "bbddee2d46d2e1643ae3623698b45b13aa2e858616d61c642f2f49e5bb14c980" - } - }, - "kubelet": { - "amd64": { - "v1.19.0": "3f03e5c160a8b658d30b34824a1c00abadbac96e62c4d01bf5c9271a2debc3ab", - "v1.19.8": "f5cad5260c29584dd370ec13e525c945866957b1aaa719f1b871c31dc30bcb3f", - "v1.19.9": "296e72c395f030209e712167fc5f6d2fdfe3530ca4c01bcd9bfb8c5e727c3d8d", - "v1.19.15": "c949b5a3fc6135b9747b592402a615ebd24791b087b3212f29248ea3ec7c62ed", - "v1.20.4": "a9f28ac492b3cbf75dee284576b2e1681e67170cd36f3f5cdc31495f1bdbf809", - "v1.20.6": "7688a663dd06222d337c8fdb5b05e1d9377e6d64aa048c6acf484bc3f2a596a8", - "v1.20.10": "de1b24f33d47cc4dc14a10f051d7d6fbbcf3800d3a07ddb45fc83660183c3a73", - "v1.21.0": "681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35", - "v1.21.1": "e77ff3ea404b2e69519ea4dce41cbdf11ae2bcba75a86d409a76eecda1c76244", - "v1.21.2": "aaf144b19c0676e1fe34a93dc753fb38f4de057a0e2d7521b0bef4e82f8ccc28", - "v1.21.3": "5bd542d656caabd75e59757a3adbae3e13d63c7c7c113d2a72475574c3c640fe", - "v1.21.4": "cdd46617d1a501531c62421de3754d65f30ad24d75beae2693688993a12bb557", - "v1.21.5": "600f70fe0e69151b9d8ac65ec195bcc840687f86ba397fce27be1faae3538a6f", - "v1.21.6": "422c29a1ba3bfeb2fc26ebd1c3596847fbbeeeef0ce2694515504513dc907813", - "v1.21.7": "59f8d7da2e994f59a369ea1705e4933949fc142bf47693e0918f4811c2e1c7b5", - "v1.21.8": "32f7eb6af9f1fd4e8b944f4f59582d455572147745e9fc04d044c383bd995c98", - "v1.21.9": "1fa0c296df6af71fca1bdd94f9fb19c7051b4b3f8cf19c353192cb96b413fcf2", - "v1.21.10": "8e0dab1cb93e61771fba594484a37a6079073ed2d707cf300c472e79b2f91bf0", - "v1.21.11": "ea22e3683016643344c5839a317b5e7b0061fdded321339a6d545766765bb10a", - "v1.21.12": "56246c4d0433a7cfd29e3e989fe3835a7545a781ff0123738713c8c78a99ec17", - "v1.21.13": "4de3bf88be86e4661f55fa69a91c3414e8e23341038b1cf366914a0794f68efb", - "v1.21.14": "ca2e5e1f2a05b86e4f758181b172eb12bf99c3cc2a2b5b3e598f4c85d4d27fda", - "v1.22.0": "fec5c596f7f815f17f5d7d955e9707df1ef02a2ca5e788b223651f83376feb7f", - "v1.22.1": "2079780ad2ff993affc9b8e1a378bf5ee759bf87fdc446e6a892a0bbd7353683", - "v1.22.2": "0fd6572e24e3bebbfd6b2a7cb7adced41dad4a828ef324a83f04b46378a8cb24", - "v1.22.3": "3f00a5f98cec024abace5bcc3580b80afc78181caf52e100fc800e588774d6eb", - "v1.22.4": "8d014cfe511d8c0a127b4e65ae2a6e60db592f9b1b512bb822490ea35958b10d", - "v1.22.5": "2be340f236a25881969eaa7d58b2279a4e31dc393cab289a74c78c0c37ba2154", - "v1.22.6": "7b009835b0ab74aa16ebf57f5179893035e0cf5994e1bcf9b783275921a0393a", - "v1.22.7": "cfc96b5f781bfbfdcb05115f4e26a5a6afc9d74bb4a5647c057b2c13086fb24d", - "v1.22.8": "2e6d1774f18c4d4527c3b9197a64ea5705edcf1b547c77b3e683458d771f3ce7", - "v1.22.9": "61530a9e6a5cb1f971295de860a8ade29db65d0dff50d1ffff3de1155dfd0c02", - "v1.22.10": "c1aa6e9f59cfc765d33b382f604140699ab97c9c4212a905d5e1bcd7ef9a5c8b", - "v1.22.11": "50fb1ede16c15dfe0bcb9fa98148d969ae8efeb8b599ce5eb5f09ab78345c9d1", - "v1.22.12": "d54539bd0fa43b43e9ad2ac4e6644bcb3f1e98b8fc371befba7ac362d93a6b00", - "v1.22.13": "f55a72f5546ecf463f54e9220a1c38179b94b32ba561dfd6ec1f2fbe8231d640", - "v1.22.14": "76b1512da1104b4e80e23fa2c4d1cbd87b865f7bc25a41a46932cf0a219469ac", - "v1.22.15": "3c00f6d4e329c40c727eaf69f46eec25879ddce87c0b21b51fa1b3c6e55218b9", - "v1.22.16": "e47125b84bc2c5d4f8bd015d7c28e7e736e03ed9c3a44c0e28fd42f2b41af749", - "v1.22.17": "48d200775000567256a8c114cf4f5d389468b175c3add6b232ec3b26f03e8564", - "v1.23.0": "4756ff345dd80704b749d87efb8eb294a143a1f4a251ec586197d26ad20ea518", - "v1.23.1": "7ff47abf62096a41005d18c6d482cf73f26b613854173327fa9f2b98720804d4", - "v1.23.2": "c3c4be17910935d234b776288461baf7a9c6a7414d1f1ac2ef8d3a1af4e41ab6", - "v1.23.3": "8f9d2dd992af82855fbac2d82e030429b08ba7775e4fee7bf043eb857dfb0317", - "v1.23.4": "ec3db57edcce219c24ef37f4a6a2eef5a1543e4a9bd15e7ecc993b9f74950d91", - "v1.23.5": "253b9db2299b09b91e4c09781ce1d2db6bad2099cf16ba210245159f48d0d5e4", - "v1.23.6": "fbb83e35f6b9f7cae19c50694240291805ca9c4028676af868306553b3e9266c", - "v1.23.7": "518f67200e853253ed6424488d6148476144b6b796ec7c6160cff15769b3e12a", - "v1.23.8": "1ba15ad4d9d99cfc3cbef922b5101492ad74e812629837ac2e5705a68cb7af1e", - "v1.23.9": "a5975920be1de0768e77ef101e4e42b179406add242c0883a7dc598f2006d387", - "v1.23.10": "c2ba75b36000103af6fa2c3955c5b8a633b33740e234931441082e21a334b80b", - "v1.23.11": "b0e6d413f9b4cf1007fcb9f0ea6460ed5273a50c945ae475c224036b0ab817f7", - "v1.23.12": "98ffa8a736d3e43debb1aa61ae71dea3671989cde5e9e44c6ee51a3d47c63614", - "v1.23.13": "4d8f796b82dbe2b89b6d587bfeedf66724526b211c75a53456d4ac4014e3dcca", - "v1.23.14": "f2bef00508790f632d035a6cfdd31539115611bfc93c5a3266ceb95bb2f27b76", - "v1.23.15": "5cf382d911c13c9cc8f770251b3a2fd9399c70ac50337874f670b9078f88231d", - "v1.23.16": "ab4c64a54c3c6de57108b7b83bdcf03413285f06adccfa2bbc5535d20a684682", - "v1.23.17": "588dde06e2515601380787cb5fcb07ae3d3403130e1a5556b013b7b7fb4ab230", - "v1.24.0": "3d98ac8b4fb8dc99f9952226f2565951cc366c442656a889facc5b1b2ec2ba52", - "v1.24.1": "fc352d5c983b0ccf47acd8816eb826d781f408d27263dd8f761dfb63e69abfde", - "v1.24.2": "13da57d32be1debad3d8923e481f30aaa46bca7030b7e748b099d403b30e5343", - "v1.24.3": "da575ceb7c44fddbe7d2514c16798f39f8c10e54b5dbef3bcee5ac547637db11", - "v1.24.4": "0f34d12aaa1b911adbf75dd63df03d0674dde921fa0571a51acd2b5b576ba0a4", - "v1.24.5": "2448debe26e90341b038d7ccfcd55942c76ef3d9db48e42ceae5e8de3fbad631", - "v1.24.6": "f8b606f542327128e404d2e66a72a40dc2ddb4175fb8e93c55effeacea60921b", - "v1.24.7": "4d24c97c924c40971412cc497145ad823e4b7b87ccda97ebced375f7e886e9e2", - "v1.24.8": "2da0b93857cf352bff5d1eb42e34d398a5971b63a53d8687b45179a78540d6d6", - "v1.24.9": "8753b9ae0c3e22f09dafdb4178492582c28874f70844de38dc43eb3fad5ca8bb", - "v1.24.10": "34b1731df37d1762662bd91f1cba57a9d2ee86296813c48c4e52a9d7955a1b9e", - "v1.24.11": "1d4849fa3aee4847b8ccecc7e14b16caeb0d507a20d0250cf57e71155248f74f", - "v1.24.12": "3105c4165cd4efea154046cac27ca087b10098c1793148fe2797b631e2897a2e", - "v1.24.13": "78cdc9b460c997babc5c8d4f45fc93c7c037df94b7004ecacaa856500ed616a6", - "v1.24.14": "bcfa6c3c38dfc9467ac88a5eb14527571fc01f1114da56c204bda082560b4706", - "v1.24.15": "577bb7424eb83b4ddedc920f4e719283cf28a8468e412c2003372c138090358f", - "v1.24.16": "af1a077cba6998d28fc6a3d07b36e25f9db2e5adae4ca20895bec55bb7c48f2a", - "v1.24.17": "babf327f3089982b271ac0acff51009eb0d649f7d1f4fbd976419ee76864f633", - "v1.25.0": "7f9183fce12606818612ce80b6c09757452c4fb50aefea5fc5843951c5020e24", - "v1.25.1": "63e38bcbc4437ce10227695f8722371ec0d178067f1031d09fe1f59b6fcf214a", - "v1.25.2": "631e31b3ec648f920292fdc1bde46053cca5d5c71d622678d86907d556efaea3", - "v1.25.3": "d5c89c5e5dae6afa5f06a3e0e653ac3b93fa9a93c775a715531269ec91a54abe", - "v1.25.4": "7f7437e361f829967ee02e30026d7e85219693432ac5e930cc98dd9c7ddb2fac", - "v1.25.5": "16b23e1254830805b892cfccf2687eb3edb4ea54ffbadb8cc2eee6d3b1fab8e6", - "v1.25.6": "8485ac4a60455b77a9b518c13b3aeb0d32338ab7e9894a0b5d217fea585cd2be", - "v1.25.7": "2e3216ac291c78d82fb8988c15d9fd4cf14e2ddd9b17ff91e3abf2e5f3e14fd9", - "v1.25.8": "3aa821165da6f1bb9fdb82a91b294b7f4abfc4fdfb21a94fa1e09a9785876516", - "v1.25.9": "0eb951237379ef023aa52deedb0df5eae54fa71caeb52bdb30a4660806bed23e", - "v1.25.10": "280515c431b8c966e475de1b953b960242549cb86f0821ad819224085b449c9b", - "v1.25.11": "4801700e29405e49a7e51cccb806decd65ca3a5068d459a40be3b4c5846b9a46", - "v1.25.12": "7aa7d0b4512e6d79ada2017c054b07aaf30d4dc0d740449364a5e2c26e2c1842", - "v1.25.13": "0399cfd7031cf5f3d7f8485b243a5ef37230e63d105d5f29966f0f81a58a8f6d", - "v1.25.14": "b9d1dbd9e7c1d3bda6249f38d7cd4f63e4188fa31cddd80d5e8ac1ce3a9a4d96", - "v1.25.15": "1136c5717df316c6d4efd96a676574825f771666b7a9148338f0079bb9412720", - "v1.25.16": "b159f4b0ce7987385902faf6b97530489a6340d728a9688c5791d8d18144b4b7", - "v1.26.0": "b64949fe696c77565edbe4100a315b6bf8f0e2325daeb762f7e865f16a6e54b5", - "v1.26.1": "8b99dd73f309ca1ac4005db638e82f949ffcfb877a060089ec0e729503db8198", - "v1.26.2": "e6dd2ee432a093492936ff8505f084b5ed41662f50231f1c11ae08ee8582a3f5", - "v1.26.3": "992d6298bd494b65f54c838419773c4976aca72dfb36271c613537efae7ab7d2", - "v1.26.4": "1e29fe7a097066cfbc1c1d2ab37f8b883c8f3fec414bafe8f2c7b960b0fb60fe", - "v1.26.5": "ad5e318ff0e81bc2bef874b2038489722cfcc117bd31726d0193056458c18bff", - "v1.26.6": "da82477404414eb342d6b93533f372aa1c41956a57517453ef3d39ebbfdf8cc2", - "v1.26.7": "2926ea2cd7fcd644d24a258bdf21e1a8cfd95412b1079914ca46466dae1d74f2", - "v1.26.8": "1c68a65a6a0c2230325e29da0cc3eaaef9bbf688a7a0bb8243b4a7ebfe0e3363", - "v1.26.9": "baa2b021ab2f90c342518e2b8981a18de7e1e6b33f11c57e3ff23d40364877a8", - "v1.26.10": "4c27b3a9f332a6762f7240d0784c64775d4db5a1b881eeae05c4561d06c267ec", - "v1.26.11": "a62953f20fa9fedff50c6c5423e68981e3382d92cf04174d5bca5f4d084de0c5", - "v1.26.12": "aed0a351b01f1e6a84a0992ef1265bb0c9994b900162c075df58d0d02517d3df", - "v1.27.0": "0b4ed4fcd75d33f5dff3ba17776e6089847fc83064d3f7a3ad59a34e94e60a29", - "v1.27.1": "cb2845fff0ce41c400489393da73925d28fbee54cfeb7834cd4d11e622cbd3a7", - "v1.27.2": "a0d12afcab3b2836de4a427558d067bebdff040e9b306b0512c93d9d2a066579", - "v1.27.3": "c0e18da6a55830cf4910ecd7261597c66ea3f8f58cf44d4adb6bdcb6e2e6f0bf", - "v1.27.4": "385f65878dc8b48df0f2bd369535ff273390518b5ac2cc1a1684d65619324704", - "v1.27.5": "66df07ab4f9d72028c97ec7e5eea23adc0ab62a209ba2285431456d7d75a5bb3", - "v1.27.6": "daa42f9b6f5e2176bbce0d24d89a05613000630bcddec1fafd2a8d42a523ce9d", - "v1.27.7": "236bc8bc22c52e914d3364c23e273628c63e193365b6a43b8cb013716c1cd2f5", - "v1.27.8": "2e0557b38c5b9a1263eed25a0b84d741453ed9c0c7bd916f80eadaf7edfb7784", - "v1.27.9": "ede60eea3acbac3f35dbb23d7b148f45cf169ebbb20af102d3ce141fc0bac60c", - "v1.28.0": "bfb6b977100963f2879a33e5fbaa59a5276ba829a957a6819c936e9c1465f981", - "v1.28.1": "2bc22332f44f8fcd3fce57879fd873f977949ebd261571fbae31fbb2713a5dd3", - "v1.28.2": "17edb866636f14eceaad58c56eab12af7ab3be3c78400aff9680635d927f1185", - "v1.28.3": "a3a058b4ba30da01ffe1801cd38fcad58a9022a2d39e080b4b2e0e9749a75ad5", - "v1.28.4": "db2a473b73c3754d4011590f2f0aa877657608499590c6b0f8b40bec96a3e9ba", - "v1.28.5": "bf37335da58182783a8c63866ec1f895b4c436e3ed96bdd87fe3f8ae8004ba1d", - "v1.29.0": "e1c38137db8d8777eed8813646b59bf4d22d19b9011ab11dc28e2e34f6b80a05" - }, - "arm64": { - "v1.19.0": "d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf", - "v1.19.8": "a00146c16266d54f961c40fc67f92c21967596c2d730fa3dc95868d4efb44559", - "v1.19.9": "796f080c53ec50b11152558b4a744432349b800e37b80516bcdc459152766a4f", - "v1.19.15": "19c2d3c8861eeebdcdeb5f84462267b6cf0584acaf8dff2f805938375c076b96", - "v1.20.4": "66bcdc7521e226e4acaa93c08e5ea7b2f57829e1a5b9decfd2b91d237e216e1d", - "v1.20.6": "6e7b44d1ca65f970b0646f7d093dcf0cfefc44d4a67f29d542fe1b7ca6dcf715", - "v1.20.10": "5107a4b2eb017039dda900cf263ec19484eee8bec070fc88803d3d9d4cc9fb18", - "v1.21.0": "17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4", - "v1.21.1": "5b37d7fc2da65a25896447685166769333b5896488de21bc9667edb4e799905e", - "v1.21.2": "525cf5506595e70bffc4c1845b3c535c7121fa2ee3daac6ca3edc69d8d63b89f", - "v1.21.3": "5d21da1145c25181605b9ad0810401545262fc421bbaae683bdb599632e834c1", - "v1.21.4": "12c849ccc627e9404187adf432a922b895c8bdecfd7ca901e1928396558eb043", - "v1.21.5": "746a535956db55807ef71772d2a4afec5cc438233da23952167ec0aec6fe937b", - "v1.21.6": "041441623c31bc6b0295342b8a2a5930d87545473e7c761ea79f3ff186c0ff52", - "v1.21.7": "02adf21a8de206cf64c4bff5723adb08377ecdcc38ff1efbfefd3abe2e415bb8", - "v1.21.8": "1d880cd437457b6a52c95fa5cfb62f05bdcea8fc29b87aaa5535a67c89a279d4", - "v1.21.9": "8797c78961cb71a757f35714d2735bb8bdbea94fc13d567bc0f1cf4f8e49e880", - "v1.21.10": "5278427751381b90299e4ef330f41ca6b691aab39c3100cd200344ce6a7481c9", - "v1.21.11": "ec0df7cf90f3422d674f9881e33d6e329a12e0f5bb438b422999493fd4370edf", - "v1.21.12": "cb523115a0aef43fc7f1de58c33d364185b3888af2083c303e6cc59335431ac2", - "v1.21.13": "4ffef9ed33067858f96c5662f61753791191eebe208a75ae263ca96270448249", - "v1.21.14": "da40431ecee2be8167d07669d06f4f7b046b582c6fc5b5e8033c5f8a14d89adc", - "v1.22.0": "cea637a7da4f1097b16b0195005351c07032a820a3d64c3ff326b9097cfac930", - "v1.22.1": "d5ffd67d8285fb224a1c49622fd739131f7b941e3d68f233dec96e72c9ebee63", - "v1.22.2": "f5fe3d6f4b2df5a794ebf325dc17fcdfe905a188e25f7c7e47d9cd15f14f8c2d", - "v1.22.3": "d0570f09bd5137ff2f672a0b177a6b78fd294a42db21f094dc02c613436ce8d1", - "v1.22.4": "c0049ab240b27a9dd57be2bb98356c62582d975ba2f790a61b34f155b12ab7e6", - "v1.22.5": "e68536cff9172d1562edddd7194d20302472a064009bf7c0ed8d79d030cb61aa", - "v1.22.6": "fbb823fe82b16c6f37911e907d3e4921f4642d5d48eb60e56aba1d7be0665430", - "v1.22.7": "8291d304c0ba4faec4336336d4cdd5159f5c90652b8b0d6be0cb5ce8f8bf92e3", - "v1.22.8": "604c672908a3b3cbbcf9d109d8d5fef0879992ddcf0d3e0766079d3bb7d0ca3e", - "v1.22.9": "d7a692ee4f5f5929a15c61947ae2deecb71b0945461f6064ced83d13094028e8", - "v1.22.10": "2376a7ecc044bc4b5cdae9a0a14d058ae5c1803450f3a8ffdce656785e9e251e", - "v1.22.11": "d20398fa95ee724d63c3263af65eeb49e56c963fcace92efed2d2d0f6084c11a", - "v1.22.12": "0e58133c153be32e8e61004cfdc18f8a02ef465f979c6d5bf3e998fbe3f89fca", - "v1.22.13": "f8c1ec9fec6b36646ac05e1e26f0cd3e20395b500eca8ee3baeb3ca59935fdb0", - "v1.22.14": "663287b907c4aed4dfde55639da15a0d23fd0608b13afa9cf71fc3da850f3660", - "v1.22.15": "0c34cbda04ae914f342e683cf70f96d56d46033457d46ea79445e1483b501565", - "v1.22.16": "e28c762714618bccced1cc94eff4b6553c0158193083855b4d9bb33beb9bd6b0", - "v1.22.17": "dd76d33b2a72693b9a88614cef72367dd737d74433fd9e65129684086719d632", - "v1.23.0": "a546fb7ccce69c4163e4a0b19a31f30ea039b4e4560c23fd6e3016e2b2dfd0d9", - "v1.23.1": "c24e4ab211507a39141d227595610383f7c5686cae3795b7d75eebbce8606f3d", - "v1.23.2": "65372ad077a660dfb8a863432c8a22cd0b650122ca98ce2e11f51a536449339f", - "v1.23.3": "95c36d0d1e65f6167f8fa80df04b3a816bc803e6bb5554f04d6af849c729a77d", - "v1.23.4": "c4f09c9031a34549fbaa48231b115fee6e170ce6832dce26d4b50b040aad2311", - "v1.23.5": "61f7e3ae0eb00633d3b5163c046cfcae7e73b5f26d4ffcf343f3a45904323583", - "v1.23.6": "11a0310e8e7af5a11539ac26d6c14cf1b77d35bce4ca74e4bbd053ed1afc8650", - "v1.23.7": "e96b746a77b00c04f1926035899a583ce28f02e9a5dca26c1bfb8251ca6a43bb", - "v1.23.8": "1b4ec707e29e8136e3516a437cb541a79c52c69b1331a7add2b47e7ac7d032e6", - "v1.23.9": "c11b14ab3fa8e567c54e893c5a937f53618b26c9b62416cc8aa7760835f68350", - "v1.23.10": "8ce1c79ee7c5d346719e3637e72a51dd96fc7f2e1f443aa39b05c1d9d9de32c8", - "v1.23.11": "ce4f568c3193e8e0895062f783980da89adb6b54a399c797656a3ce172ddb2fc", - "v1.23.12": "b802f12c79a9797f83a366c617144d019d2994fc724c75f642a9d031ce6a3488", - "v1.23.13": "4e2297c9893d425bfcd80741b95fb1a5b59b4fd4f4bcf782ccab94760e653cdf", - "v1.23.14": "80cdff15398c8215bb7337efdee25b40c862befbdf7925f6a8aca71bc9a79eae", - "v1.23.15": "b5540d2b67f325ad79af6b86a88bc3d1a8a225453911e7ebb7387788ce355a87", - "v1.23.16": "8ebf22503229b14e29136de68bd2b6994f70433dd22e0be9ee9bcb785f977a00", - "v1.23.17": "95e25ee4d2f34f628ba42685c6ae1ba6efdf86f2e76b5cceb0b48de4d66522a7", - "v1.24.0": "8f066c9a048dd1704bf22ccf6e994e2fa2ea1175c9768a786f6cb6608765025e", - "v1.24.1": "c2189c6956afda0f6002839f9f14a9b48c89dcc0228701e84856be36a3aac6bf", - "v1.24.2": "40a8460e104fbf97abee9763f6e1f2143debc46cc6c9a1a18e21c1ff9960d8c0", - "v1.24.3": "6c04ae25ee9b434f40e0d2466eb4ef5604dc43f306ddf1e5f165fc9d3c521e12", - "v1.24.4": "2d9817c1e9e1edd9480aa05862ea6e9655a9512d820b1933175f5d7c8253ca61", - "v1.24.5": "dd5dcea80828979981654ec0732b197be252a3259a527cbc299d9575bc2de3e8", - "v1.24.6": "2a7b8e131d6823462e38bc1514b5dea5dca86254b3a12ed4a0fa653c2e06dd0e", - "v1.24.7": "d8bd38e595ca061c53d3b7d1daebe5b3cc1ad44c731666bd5e842d336077db4b", - "v1.24.8": "4e1427651e4ff3927f96ce4b93c471ccc76c683fc1619ee0d677d77345b54edb", - "v1.24.9": "34021c6cf593ffc4361e9e2adc3d6e4f5683383eeb894f40d63ccfa268e84f4c", - "v1.24.10": "e4f5f6d864dfcbe5e5773039f4c44b026a1fa8626f459c82c00e8a4d85bbc089", - "v1.24.11": "43a535958b7b9d8da5600208064eb3f799d9069ced6d836a79d24dd87525f66f", - "v1.24.12": "b4116fb92694d75afd2ac3be742fe3c020470e53546cfacc73eb8a4e33cfcab1", - "v1.24.13": "4449135fb486c275ebf928ad0873c391297a1d78bd871163f37920bc41d1d171", - "v1.24.14": "421665d86cbebde3a5e1a58463ac23ab2874aa42a9a8d8e1ca2cadfca9de5384", - "v1.24.15": "a1e253b03c1a6e22f315e4619d4cf26ee1282ccb3ef94c37357ee930699a10d7", - "v1.24.16": "3cc22ae7cb1768031fe17a672898cb625bd8cf9e679282953b805df780718654", - "v1.24.17": "dc83bc7a47715f1fa29ea7f1e1d13e3626a8bfbe3a6793d8cb97273289fe9bba", - "v1.25.0": "69572a7b3d179d4a479aa2e0f90e2f091d8d84ef33a35422fc89975dc137a590", - "v1.25.1": "b6baa99b99ecc1f358660208a9a27b64c65f3314ff95a84c73091b51ac98484b", - "v1.25.2": "c9348c0bae1d723a39235fc041053d9453be6b517082f066b3a089c3edbdd2ae", - "v1.25.3": "929d25fc3f901749b058141a9c624ff379759869e09df49b75657c0be3141091", - "v1.25.4": "8ff80a12381fad2e96c9cec6712591018c830cdd327fc7bd825237aa51a6ada3", - "v1.25.5": "18aa53ff59740a11504218905b51b29cc78fb8b5dd818a619141afa9dafb8f5a", - "v1.25.6": "6dade59b6fe4b94f03ee173692f5713e023b0cd1abaa8f5ebe4263b49a63df38", - "v1.25.7": "cb9176563c7a75be1e8ea23d8e366ced97becabd4626fde01620ec71d2eb1fc2", - "v1.25.8": "6c995b05b54cc0ce4eb6bf3097565167069b2ce45ba965972430c631c467d239", - "v1.25.9": "5af11a2948c87076540bb78ee99563b373301c9d8bb694395e2ce7fb14a76344", - "v1.25.10": "16ad59fb38337b19afa29ca7d05ccd61e859599c598445d0e11c760fc1086e6b", - "v1.25.11": "0140cf3aee0b9386fc8430c32bc94c169a6e50640947933733896e01490cbf6c", - "v1.25.12": "3402d0fcec5105bb08b917bb5a29a979c674aa10a12a1dfe4e0d80b292f9fe56", - "v1.25.13": "7a29aabb40a984f104b88c09312e897bb710e6bb68022537f8700e70971b984b", - "v1.25.14": "3a3d4ac26b26baef43188a6f52d40a20043db3ffdbcbefab8be222b58ce0f713", - "v1.25.15": "9ca686d5fac093bd3dfe72e8614a5d8d482b7e22d6a78ff5a2a639fc54e603b6", - "v1.25.16": "5f379fc59db0efc288236dbd0abd32b1b0206d1c435001b9c0c3996171e20ffd", - "v1.26.0": "fb033c1d079cac8babb04a25abecbc6cc1a2afb53f56ef1d73f8dc3b15b3c09e", - "v1.26.1": "f4b514162b52d19909cf0ddf0b816d8d7751c5f1de60eda90cd84dcccc56c399", - "v1.26.2": "33e77f93d141d3b9e207ae50ff050186dea084ac26f9ec88280f85bab9dad310", - "v1.26.3": "d360f919c279a05441b27178030c3d17134c1f257c95f4b22bdb28c2290993e7", - "v1.26.4": "a925a5d20d29c362f0c4d60cb005f21d44576837510e0bc65c817961969b4e7e", - "v1.26.5": "4256e46eb36bea3c31b0372c4d5b669964a2cfb1eabb7e0e2e0dcb1cdd81f2e8", - "v1.26.6": "44c2cd64e1317df8252bca1cf196227c543005a3b10d52fb114401cb1617f32f", - "v1.26.7": "73e086cfd8cd1cef559e739e19aff2932f8a9e0bdba3c9faeb9185a86d067fbb", - "v1.26.8": "0f15e484c4a7a7c3bad9e0aa4d4334ca029b97513fbe03f053201dd937cf316e", - "v1.26.9": "f6b1dcee9960ffe6b778dc91cabef8ce4a7bd06c76378ef2784232709eace6a5", - "v1.26.10": "ddebcc1af7f203a2ee3d80dad0baaf84a4680748839f5583b39cbce4b8afa7f2", - "v1.26.11": "ff8940394446028e75a2b8155e22eccf635f6a128f45dee41e293493d2743d17", - "v1.26.12": "d4406ed5bfd12768c03fc4fbe011a01e5c91b74d1d4b526fe3ac320d13295ffb", - "v1.27.0": "37aa2edc7c0c4b3e488518c6a4b44c8aade75a55010534ee2be291220c73d157", - "v1.27.1": "dbb09d297d924575654db38ed2fc627e35913c2d4000c34613ac6de4995457d0", - "v1.27.2": "810cd9a611e9f084e57c9ee466e33c324b2228d4249ff38c2588a0cc3224f10d", - "v1.27.3": "2838fd55340d59f777d7bd7e5989fc72b7a0ca198cf4f3f723cd9956859ce942", - "v1.27.4": "c75ad8e7c7ef05c0c021b21a9fe86e92f64db1e4c1bc84e1baf45d8dbb8ba8d1", - "v1.27.5": "4e78fafdeb5d61ab6ebcd6e75e968c47001c321bec169bb9bd9f001132de5321", - "v1.27.6": "be579ef4e8fa3e1de9d40a77e4d35d99e535a293f66bf3038cbea9cf803d11e5", - "v1.27.7": "ed5bfa48ee64d5e6cf23ed9fc03ea0593021839429fdc1ea7cc2ebf3f11b6491", - "v1.27.8": "71849182ceb018dc084f499ad28b7b1afb7f23e35ccaf8421941dd5dafef0d4c", - "v1.27.9": "8a14bc3739f5ca3b23d08301c2e769ee58c8d1cecb7243b46b1c098ae77effd7", - "v1.28.0": "05dd12e35783cab4960e885ec0e7d0e461989b94297e7bea9018ccbd15c4dce9", - "v1.28.1": "9b7fa64b2785da4a38768377961e227f8da629c56a5df43ca1b665dd07b56f3c", - "v1.28.2": "32269e9ec38c561d028b65c3048ea6a100e1292cbe9e505565222455c8096577", - "v1.28.3": "64f56e9c55183919153fe59df2c9015dff09c56de13a3cbccc0f04a95b76dab9", - "v1.28.4": "bf203989dd9b3987b8a0d2331dcce6319f834b57df810fafba5a4805d54823ac", - "v1.28.5": "28ddb696eb6e076f2a2f59ccaa2e409785a63346e5bda819717c6e0f58297702", - "v1.29.0": "0e0e4544c2a0a3475529154b7534d0d58683466efa04a2bb2e763b476db0bb16" - } - }, - "kubectl": { - "amd64": { - "v1.19.0": "79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f", - "v1.19.8": "a0737d3a15ca177816b6fb1fd59bdd5a3751bfdc66de4e08dffddba84e38bf3f", - "v1.19.9": "7128c9e38ab9c445a3b02d3d0b3f0f15fe7fbca56fd87b84e575d7b29e999ad9", - "v1.19.15": "6f2ac7db8cfd59f660abc9891c1bb7da2dabd1cf5e114d836f2ffd39ee677d04", - "v1.20.4": "98e8aea149b00f653beeb53d4bd27edda9e73b48fed156c4a0aa1dabe4b1794c", - "v1.20.6": "89ae000df6bbdf38ae4307cc4ecc0347d5c871476862912c0a765db9bf05284e", - "v1.20.10": "1e87edb99b7a92a142b458976ae75412d3ee22421793968b03213ddd007c0530", - "v1.21.0": "9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0", - "v1.21.1": "58785190e2b4fc6891e01108e41f9ba5db26e04cebb7c1ac639919a931ce9233", - "v1.21.2": "55b982527d76934c2f119e70bf0d69831d3af4985f72bb87cd4924b1c7d528da", - "v1.21.3": "631246194fc1931cb897d61e1d542ef2321ec97adcb859a405d3b285ad9dd3d6", - "v1.21.4": "9410572396fb31e49d088f9816beaebad7420c7686697578691be1651d3bf85a", - "v1.21.5": "060ede75550c63bdc84e14fcc4c8ab3017f7ffc032fc4cac3bf20d274fab1be4", - "v1.21.6": "810eadc2673e0fab7044f88904853e8f3f58a4134867370bf0ccd62c19889eaa", - "v1.21.7": "d25d6b6f67456cc059680e7443c424eb613d9e840850a7be5195cff73fed41b8", - "v1.21.8": "84eaef3da0b508666e58917ebe9a6b32dcc6367bddf6e4489b909451877e3e70", - "v1.21.9": "195d5387f2a6ca7b8ab5c2134b4b6cc27f29372f54b771947ba7c18ee983fbe6", - "v1.21.10": "24ce60269b1ffe1ca151af8bfd3905c2427ebef620bc9286484121adf29131c0", - "v1.21.11": "9c45ce24ad412701beeac8d9f0004787209d76dd66390915f38a8682358484cb", - "v1.21.12": "5a8bde5198dc0e87dfa8ebc50c29f69becdc94c756254f6b2c3f37cdbfaf2e42", - "v1.21.13": "24fc367b5add5a06713ea8103041f6fc0cf4560a17f2c17916e7930037adc84a", - "v1.21.14": "0c1682493c2abd7bc5fe4ddcdb0b6e5d417aa7e067994ffeca964163a988c6ee", - "v1.22.0": "703e70d49b82271535bc66bc7bd469a58c11d47f188889bd37101c9772f14fa1", - "v1.22.1": "78178a8337fc6c76780f60541fca7199f0f1a2e9c41806bded280a4a5ef665c9", - "v1.22.2": "aeca0018958c1cae0bf2f36f566315e52f87bdab38b440df349cd091e9f13f36", - "v1.22.3": "0751808ca8d7daba56bf76b08848ef5df6b887e9d7e8a9030dd3711080e37b54", - "v1.22.4": "21f24aa723002353eba1cc2668d0be22651f9063f444fd01626dce2b6e1c568c", - "v1.22.5": "fcb54488199c5340ff1bc0e8641d0adacb27bb18d87d0899a45ddbcc45468611", - "v1.22.6": "1ab07643807a45e2917072f7ba5f11140b40f19675981b199b810552d6af5c53", - "v1.22.7": "4dd14c5b61f112b73a5c9c844011a7887c4ffd6b91167ca76b67197dee54d388", - "v1.22.8": "761bf1f648056eeef753f84c8365afe4305795c5f605cd9be6a715483fe7ca6b", - "v1.22.9": "ae6a9b585f9a366d24bb71f508bfb9e2bb90822136138109d3a91cd28e6563bb", - "v1.22.10": "225bc8d4ac86e3a9e36b85d2d9cb90cd4b4afade29ba0292f47834ecf570abf2", - "v1.22.11": "a61c697e3c9871da7b609511248e41d9c9fb6d9e50001425876676924761586b", - "v1.22.12": "8e36c8fa431e454e3368c6174ce3111b7f49c28feebdae6801ab3ca45f02d352", - "v1.22.13": "b96d2bc9137ec63546a29513c40c5d4f74e9f89aa11edc15e3c2f674d5fa3e02", - "v1.22.14": "a4408b32b9729e38c14b38a64ea6f00d67d2127f9c1314fbc2273a37a987a2d2", - "v1.22.15": "239a48f1e465ecfd99dd5e3d219066ffea7bbd4cdedb98524e82ff11fd72ba12", - "v1.22.16": "12da5128e2377f9c9eb97b854c95445e00839437de0543968bd3a54b5ae596d8", - "v1.22.17": "7506a0ae7a59b35089853e1da2b0b9ac0258c5309ea3d165c3412904a9051d48", - "v1.23.0": "2d0f5ba6faa787878b642c151ccb2c3390ce4c1e6c8e2b59568b3869ba407c4f", - "v1.23.1": "156fd5e7ebbedf3c482fd274089ad75a448b04cf42bc53f370e4e4ea628f705e", - "v1.23.2": "5b55b58205acbafa7f4e3fc69d9ce5a9257be63455db318e24db4ab5d651cbde", - "v1.23.3": "d7da739e4977657a3b3c84962df49493e36b09cc66381a5e36029206dd1e01d0", - "v1.23.4": "3f0398d4c8a5ff633e09abd0764ed3b9091fafbe3044970108794b02731c72d6", - "v1.23.5": "715da05c56aa4f8df09cb1f9d96a2aa2c33a1232f6fd195e3ffce6e98a50a879", - "v1.23.6": "703a06354bab9f45c80102abff89f1a62cbc2c6d80678fd3973a014acc7c500a", - "v1.23.7": "b4c27ad52812ebf3164db927af1a01e503be3fb9dc5ffa058c9281d67c76f66e", - "v1.23.8": "299803a347e2e50def7740c477f0dedc69fc9e18b26b2f10e9ff84a411edb894", - "v1.23.9": "053561f7c68c5a037a69c52234e3cf1f91798854527692acd67091d594b616ce", - "v1.23.10": "3ffa658e7f1595f622577b160bdcdc7a5a90d09d234757ffbe53dd50c0cb88f7", - "v1.23.11": "cf04ad2fa1cf118a951d690af0afbbe8f5fc4f02c721c848080d466e6159111e", - "v1.23.12": "b150c7c4830cc3be4bedd8998bf36a92975c95cd1967b4ef2d1edda080ffe5d9", - "v1.23.13": "fae6957e6a7047ad49cdd20976cd2ce9188b502c831fbf61f36618ea1188ba38", - "v1.23.14": "13ce4b18ba6e15d5d259249c530637dd7fb9722d121df022099f3ed5f2bd74cd", - "v1.23.15": "adab29cf67e04e48f566ce185e3904b5deb389ae1e4d57548fcf8947a49a26f5", - "v1.23.16": "5f914edc9dbfbe1b8b8dc0f5dbbac28720a8dffeb940e3339c371e3612c37e48", - "v1.23.17": "f09f7338b5a677f17a9443796c648d2b80feaec9d6a094ab79a77c8a01fde941", - "v1.24.0": "94d686bb6772f6fb59e3a32beff908ab406b79acdfb2427abdc4ac3ce1bb98d7", - "v1.24.1": "0ec3c2dbafc6dd27fc8ad25fa27fc527b5d7356d1830c0efbb8adcf975d9e84a", - "v1.24.2": "f15fb430afd79f79ef7cf94a4e402cd212f02d8ec5a5e6a7ba9c3d5a2f954542", - "v1.24.3": "8a45348bdaf81d46caf1706c8bf95b3f431150554f47d444ffde89e8cdd712c1", - "v1.24.4": "4a76c70217581ba327f0ad0a0a597c1a02c62222bb80fbfea4f2f5cb63f3e2d8", - "v1.24.5": "3037f2ec62956e7146fc86defb052d8d3b28e2daa199d7e3ff06d1e06a6286ed", - "v1.24.6": "3ba7e61aecb19eadfa5de1c648af1bc66f5980526645d9dfe682d77fc313b74c", - "v1.24.7": "2d88e56d668b1d7575b4783f22d512e94da432f42467c3aeac8a300b6345f12d", - "v1.24.8": "f93c18751ec715b4d4437e7ece18fe91948c71be1f24ab02a2dde150f5449855", - "v1.24.9": "7e13f33b7379b6c25c3ae055e4389eb3eef168e563f37b5c5f1be672e46b686e", - "v1.24.10": "d8e9cd9bb073ff09e2f2a74cf48e94a9b9d4f2fa2e2dd91b68b01f64e7061a3b", - "v1.24.11": "c8bdf1b12d5ac91d163c07e61b9527ef718bec6a00f4fd4cf071591218f59be5", - "v1.24.12": "25875551d4242339bcc8cef0c18f0a0f631ea621f6fab1190a5aaab466634e7c", - "v1.24.13": "52455fe485fa11c650ab07fea2e4856b5ea5a3e6ef8a1b3b6121e6361437fff9", - "v1.24.14": "3ef141cb84803fbdcea7db270e9d6678ba064c2c9f2eccce91adedf09daba706", - "v1.24.15": "a45d390e17d1cd4bf93eaa733ebc87ed1a38fd867316919bdc59bee9f96e4d8e", - "v1.24.16": "33f8179cd124ab97268ec0cf4c91a05514c8e82d7a341d337e92881401844d71", - "v1.24.17": "3e9588e3326c7110a163103fc3ea101bb0e85f4d6fd228cf928fa9a2a20594d5", - "v1.25.0": "e23cc7092218c95c22d8ee36fb9499194a36ac5b5349ca476886b7edc0203885", - "v1.25.1": "9cc2d6ce59740b6acf6d5d4a04d4a7d839b0a81373248ef0ce6c8d707143435b", - "v1.25.2": "8639f2b9c33d38910d706171ce3d25be9b19fc139d0e3d4627f38ce84f9040eb", - "v1.25.3": "f57e568495c377407485d3eadc27cda25310694ef4ffc480eeea81dea2b60624", - "v1.25.4": "e4e569249798a09f37e31b8b33571970fcfbdecdd99b1b81108adc93ca74b522", - "v1.25.5": "6a660cd44db3d4bfe1563f6689cbe2ffb28ee4baf3532e04fff2d7b909081c29", - "v1.25.6": "ba876aef0e9d7e2e8fedac036ec194de5ec9b6d2953e30ff82a2758c6ba32174", - "v1.25.7": "6cdbaf3fdd1032fc8e560ccc0a75b5bd6fa5b6cb45491e9677872f511131ad3d", - "v1.25.8": "80e70448455f3d19c3cb49bd6ff6fc913677f4f240d368fa2b9f0d400c8cd16e", - "v1.25.9": "aaa5ea3b3630730d2b8a8ef3cccb14b47754602c7207c7b0717158ae83c7cb10", - "v1.25.10": "62129056c9e390b23253aadfce1fe23e43316cb3d79a73303d687d86d73707f2", - "v1.25.11": "d12bc7d26313546827683ff7b79d0cb2e7ac17cdad4dce138ed518e478b148a7", - "v1.25.12": "75842752ea07cb8ee2210df40faa7c61e1317e76d5c7968e380cae83447d4a0f", - "v1.25.13": "22c5d5cb95b671ea7d7accd77e60e4a787b6d40a6b8ba4d6c364cb3ca818c29a", - "v1.25.14": "06351e043b8ecd1206854643a2094ccf218180c1b3fab5243f78d2ccfc630ca2", - "v1.25.15": "6428297af0b06d1bb87601258fb61c13d82bf3187b2329b5f38b6f0fec5be575", - "v1.25.16": "5a9bc1d3ebfc7f6f812042d5f97b82730f2bdda47634b67bddf36ed23819ab17", - "v1.26.0": "b6769d8ac6a0ed0f13b307d289dc092ad86180b08f5b5044af152808c04950ae", - "v1.26.1": "d57be22cfa25f7427cfb538cfc8853d763878f8b36c76ce93830f6f2d67c6e5d", - "v1.26.2": "fcf86d21fb1a49b012bce7845cf00081d2dd7a59f424b28621799deceb5227b3", - "v1.26.3": "026c8412d373064ab0359ed0d1a25c975e9ce803a093d76c8b30c5996ad73e75", - "v1.26.4": "636ac0eaa467dbceda4b2c4e33662adc9709f5ce40341c9fc1a687fc276ac02d", - "v1.26.5": "5080bb2e9631fe095139f7e973df9a31eb73e668d1785ffeb524832aed8f87c3", - "v1.26.6": "ee23a539b5600bba9d6a404c6d4ea02af3abee92ad572f1b003d6f5a30c6f8ab", - "v1.26.7": "d9dc7741e5f279c28ef32fbbe1daa8ebc36622391c33470efed5eb8426959971", - "v1.26.8": "d8e0dba258d1096f95bb6746ca359db2ee8abe226e777f89dc8a5d1bb76795aa", - "v1.26.9": "98ea4a13895e54ba24f57e0d369ff6be0d3906895305d5390197069b1da12ae2", - "v1.26.10": "93ad44b4072669237247bfbc171be816f08e7e9e4260418d2cfdd0da1704ae86", - "v1.26.11": "27c34a0870230d9dd723e1e01114634e396cd2a3d25ced263b769a4bd53e4edd", - "v1.26.12": "8e6af8d68e7b9d2a1eb43255c0da793276e549a34a2b9c3c87a9c26438e7fd71", - "v1.27.0": "71a78259d70da9c5540c4cf4cff121f443e863376f68f89a759d90cef3f51e87", - "v1.27.1": "7fe3a762d926fb068bae32c399880e946e8caf3d903078bea9b169dcd5c17f6d", - "v1.27.2": "4f38ee903f35b300d3b005a9c6bfb9a46a57f92e89ae602ef9c129b91dc6c5a5", - "v1.27.3": "fba6c062e754a120bc8105cde1344de200452fe014a8759e06e4eec7ed258a09", - "v1.27.4": "4685bfcf732260f72fce58379e812e091557ef1dfc1bc8084226c7891dd6028f", - "v1.27.5": "9a091fb65e4cf4e8be3ce9a21c79210177dd7ce31a2998ec638c92f37f058bcd", - "v1.27.6": "2b7adb71c8630904da1b94e262c8c3c477e9609b3c0ed8ae1213a1e156ae38dd", - "v1.27.7": "e5fe510ba6f421958358d3d43b3f0b04c2957d4bc3bb24cf541719af61a06d79", - "v1.27.8": "027b3161e99fa0a7fa529e8f17f73ee2c0807c81c721ca7cf307f6b41c17bc57", - "v1.27.9": "d0caae91072297b2915dd65f6ef3055d27646dce821ec67d18da35ba9a8dc85b", - "v1.28.0": "4717660fd1466ec72d59000bb1d9f5cdc91fac31d491043ca62b34398e0799ce", - "v1.28.1": "e7a7d6f9d06fab38b4128785aa80f65c54f6675a0d2abef655259ddd852274e1", - "v1.28.2": "c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5", - "v1.28.3": "0c680c90892c43e5ce708e918821f92445d1d244f9b3d7513023bcae9a6246d1", - "v1.28.4": "893c92053adea6edbbd4e959c871f5c21edce416988f968bec565d115383f7b8", - "v1.28.5": "2a44c0841b794d85b7819b505da2ff3acd5950bd1bcd956863714acc80653574", - "v1.29.0": "0e03ab096163f61ab610b33f37f55709d3af8e16e4dcc1eb682882ef80f96fd5" - }, - "arm64": { - "v1.19.0": "d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f", - "v1.19.8": "8f037ab2aa798bbc66ebd1d52653f607f223b07813bcf98d9c1d0c0e136910ec", - "v1.19.9": "628627d01c9eaf624ffe3cf1195947a256ea5f842851e42682057e4233a9e283", - "v1.19.15": "3c3179888b516a4dadf7aa25e5c93577a2f763031e615c56c78f8413bc706f4d", - "v1.20.4": "0fd64b3e5d3fda4637c174a5aea0119b46d6cbede591a4dc9130a81481fc952f", - "v1.20.6": "1d0a29420c4488b15adb44044b193588989b95515cd6c8c03907dafe9b3d53f3", - "v1.20.10": "e559bcf16c824a2337125f20a2d64bfbf3959c713aa4f711871a694e2f58d4d8", - "v1.21.0": "a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d", - "v1.21.1": "d7e1163f4127efd841e5f5db6eacced11c2a3b20384457341b19ca295d0c535f", - "v1.21.2": "5753051ed464d0f1af05a3ca351577ba5680a332d5b2fa7738f287c8a40d81cf", - "v1.21.3": "2be58b5266faeeb93f38fa72d36add13a950643d2ae16a131f48f5a21c66ef23", - "v1.21.4": "8ac78de847118c94e2d87844e9b974556dfb30aff0e0d15fd03b82681df3ac98", - "v1.21.5": "fca8de7e55b55cceab9902aae03837fb2f1e72b97aa09b2ac9626bdbfd0466e4", - "v1.21.6": "a193997181cdfa00be0420ac6e7f4cfbf6cedd6967259c5fda1d558fa9f4efe0", - "v1.21.7": "50e5d76831af7b83228a5191ae10313c33639d03fadd89ad3cd492d280be4f88", - "v1.21.8": "ec122a1c239798c8a233377113b71bed808191dd931137f0631faa2d91fddb2a", - "v1.21.9": "6e2893b5de590fd9587ba327c048e5318e9e12e2acdc5a83c995c57ae822e6e4", - "v1.21.10": "d0a88f897824954ec104895eae5f9ff9a173b162d1c9245c274cfe8db323fb37", - "v1.21.11": "2d51a37128d823520f5f2b70436f5e3ae426eeacd16d671ae7806d421e4f57d8", - "v1.21.12": "3f3739cff2d1a4c28d2f89d06a2bd39388af95ce25f70b6d5cc0de0538d2ce4b", - "v1.21.13": "ca40722f3a3cb1b7687e5cdbd3a374b64ba4566e979e8ee6cd6023fa09f82ffe", - "v1.21.14": "a23151bca5d918e9238546e7af416422b51cda597a22abaae5ca50369abfbbaa", - "v1.22.0": "8d9cc92dcc942f5ea2b2fc93c4934875d9e0e8ddecbde24c7d4c4e092cfc7afc", - "v1.22.1": "5c7ef1e505c35a8dc0b708f6b6ecdad6723875bb85554e9f9c3fe591e030ae5c", - "v1.22.2": "c5bcc7e5321d34ac42c4635ad4f6fe8bd4698e9c879dc3367be542a0b301297b", - "v1.22.3": "ebeac516cc073cfe9550f114ca326f762d958cb91a33c8c9d03ede6ba94a6088", - "v1.22.4": "3fcec0284c0fdfc22e89a5b73ebd7f51120cc3505a11a4f6d6f819d46a40b26a", - "v1.22.5": "a122ef299d75c0bec1dc1e28670d358e13743144e68223c8178462ba5c436e1d", - "v1.22.6": "b43199fe66a58f292f2c685b922330819190eb22ac41cc5c10c33fdf9f2bbc29", - "v1.22.7": "44342131947bc61e6b03103e7e1302d16fa3e5b2e2cd67e27194f66223ecf798", - "v1.22.8": "48105735b74e941a84dec6bd53637c023ad53dc5fadd9bf616347cb339c76b47", - "v1.22.9": "33724bed4dddf4d8ecd6ae75667552d121e2fb575ff2db427ce66516e048edac", - "v1.22.10": "6ce1a1315225d7d62f7d17083c9f87d4f3f5684c80da108799c99780ad520cb3", - "v1.22.11": "35da77af0581740aa8815c461ee912181fbb4cec09c2e0c9f6dbee58a48758a6", - "v1.22.12": "7d6507ecb8061f7d94d1bd6b982c56b1a1f929427bcc27a962fe66c61100f12a", - "v1.22.13": "e3e845bac0e1c30de20438433a8d75c64c237892245887a2818bd877b9601b41", - "v1.22.14": "942c5f0e7be658ed047d8691df3f80cf1dd00a642fb7eab5c5367cb2f8e937e6", - "v1.22.15": "206ccaa283eaf02f72d19cf7a490b71c7b9fa77c0eb265006db31ab84b56eac0", - "v1.22.16": "70b9cd23bde136f22510f75216ce9e97944b595f2f5193eca63ffab7bfbe683e", - "v1.22.17": "8fc2f8d5c80a6bf60be06f8cf28679a05ce565ce0bc81e70aaac38e0f7da6259", - "v1.23.0": "1d77d6027fc8dfed772609ad9bd68f611b7e4ce73afa949f27084ad3a92b15fe", - "v1.23.1": "c0c24c7f6a974390e15148a575c84878e925f32328ff96ae173ec762678e4524", - "v1.23.2": "6e7bb8ddc5fc8fa89a4c31aba02942718b092a5107585bd09a83c95039c7510b", - "v1.23.3": "6708d7a701b3d9ab3b359c6be27a3012b1c486fa1e81f79e5bdc71ffca2c38f9", - "v1.23.4": "aa45dba48791eeb78a994a2723c462d155af4e39fdcfbcb39ce9c96f604a967a", - "v1.23.5": "15cd560c04def7bbe5ee3f6f75e2cfd3913371c7e76354f4b2d5d6f536b70e39", - "v1.23.6": "4be771c8e6a082ba61f0367077f480237f9858ef5efe14b1dbbfc05cd42fc360", - "v1.23.7": "5d59447a5facd8623a79c2a296a68a573789d2b102b902aafb3a730fc4bb0d3b", - "v1.23.8": "b293fce0b3dec37d3f5b8875b8fddc64e02f0f54f54dd7742368973c52530890", - "v1.23.9": "66659f614d06d0fe80c5eafdba7073940906de98ea5ee2a081d84fa37d8c5a21", - "v1.23.10": "d88b7777b3227dd49f44dbd1c7b918f9ddc5d016ecc47547a717a501fcdc316b", - "v1.23.11": "9416cc7abaf03eb83f854a45a41986bf4e1232d129d7caafc3101a01ca11b0e3", - "v1.23.12": "88ebbc41252b39d49ce574a5a2bb25943bb82e55a252c27fe4fc096ce2dbb437", - "v1.23.13": "950626ae35fca6c26096f97cac839d76e2f29616048ad30cec68f1ff003840f2", - "v1.23.14": "857716aa5cd24500349e5de8238060845af34b91ac4683bd279988ad3e1d3efa", - "v1.23.15": "f619f8b4811d60edef692f1d888609cc279a7d8223e50e1c0dc959c7b9250e79", - "v1.23.16": "a304cb4d5815e05a45f00ac9fccfae32761a5a3e26cae40c2791b28b0b80db2d", - "v1.23.17": "c4a48fdc6038beacbc5de3e4cf6c23639b643e76656aabe2b7798d3898ec7f05", - "v1.24.0": "449278789de283648e4076ade46816da249714f96e71567e035e9d17e1fff06d", - "v1.24.1": "b817b54183e089494f8b925096e9b65af3a356d87f94b73929bf5a6028a06271", - "v1.24.2": "5a4c3652f08b4d095b686e1323ac246edbd8b6e5edd5a2626fb71afbcd89bc79", - "v1.24.3": "bdad4d3063ddb7bfa5ecf17fb8b029d5d81d7d4ea1650e4369aafa13ed97149a", - "v1.24.4": "0aa4a08ff81efe3fc1a8ef880ca2f8622e3b1f93bf622583d7b9bfe3124afe61", - "v1.24.5": "a5e348758c0f2b22adeb1b663b4b66781bded895d8ea2a714eb1de81fb00907a", - "v1.24.6": "2f62e55960b02bb63cbc9154141520ac7cf0c2d55b45dd4a72867971e24a7219", - "v1.24.7": "4b138a11b13210ce1731e06918f8fff6709c004c6fb6bec28544713854de9fe8", - "v1.24.8": "b8ac2abfcb1fa04695d18098558ff483ec2c2488877b5abc4035a543544cdcb1", - "v1.24.9": "f59c522cf5f9db826c64f28364946acb6bcb6957669291fa29b926b7812b5bbe", - "v1.24.10": "3a0fa419264b3df077bc8220d721d6cbef6cc944014d9337adfd4434ac23bebf", - "v1.24.11": "b333f110a0408448b8ba18aa41ca40571056c78e496762ee8084e741b685e206", - "v1.24.12": "a945095ceabc2b6f943c8c7c8484925b1b205738231fe7d34368a3e77dfe319b", - "v1.24.13": "750a2ee55b835994cda5f262bfa18debdcdb0d7a450875dca4f83c7cfffda353", - "v1.24.14": "df2859e4cb868926591d93c49e185048c8ed3db026d32f817367c7f8db6914f9", - "v1.24.15": "49f794a719d2e5c6cc1fbd19c45603e5e62e828be216975e4ae1a71eacba78f2", - "v1.24.16": "d7f28d3d99cee6c2cdf03e77c063aeb805b120e65c2780baf3f040c9187d6142", - "v1.24.17": "66885bda3a202546778c77f0b66dcf7f576b5a49ff9456acf61329da784a602d", - "v1.25.0": "24db547bbae294c5c44f2b4a777e45f0e2f3d6295eace0d0c4be2b2dfa45330d", - "v1.25.1": "73602eabf20b877f88642fafcbe1eda439162c2c1dbcc9ed09fdd4d7ac9919ea", - "v1.25.2": "b26aa656194545699471278ad899a90b1ea9408d35f6c65e3a46831b9c063fd5", - "v1.25.3": "cfd5092ce347a69fe49c93681a164d9a8376d69eef587da894207c62ec7d6a5d", - "v1.25.4": "a8e9cd3c6ca80b67091fc41bc7fe8e9f246835925c835823a08a20ed9bcea1ba", - "v1.25.5": "7bc650f28a5b4436df2abcfae5905e461728ba416146beac17a2634fa82a6f0a", - "v1.25.6": "1a4e2850e94d44039c73eae7a6e005b3e1435c00a62bd58df7643bdeb8475cfd", - "v1.25.7": "2c60befa0fefd3bb601e9aa0fc81ae6fb204b514849fe7fa30bea0285449a84b", - "v1.25.8": "28cf5f666cb0c11a8a2b3e5ae4bf93e56b74ab6051720c72bb231887bfc1a7c6", - "v1.25.9": "741e65b681a22074aaf9459b57dbcef6a9e993472b3019a87f57c191bc68575f", - "v1.25.10": "d5ade4f3962dc89ac80fb47010231f79b3f83b2c9569183941c0189157e514fa", - "v1.25.11": "2eb5109735c1442dd3b91a15ff74e24748efd967a3d7bf1a2b16e7aa78400677", - "v1.25.12": "315a1515b7fe254d7aa4f5928007b4f4e586bfd91ea6cbf392718099920dcb8a", - "v1.25.13": "90bb3c9126b64f5eee2bef5a584da8bf0a38334e341b427b6986261af5f0d49b", - "v1.25.14": "a52ec9119e390ad872a74fc560a6569b1758a4217fd2b03e966f77aaa2a2b706", - "v1.25.15": "ae213606b3965872b4e97ceb58fce5be796e7b26ea680681e8a3c2b549fe1701", - "v1.25.16": "d6c23c80828092f028476743638a091f2f5e8141273d5228bf06c6671ef46924", - "v1.26.0": "79b14e4ddada9e81d2989f36a89faa9e56f8abe6e0246e7bdc305c93c3731ea4", - "v1.26.1": "4027cb0a2840bc14ec3f18151b3360dd2d1f6ce730ed5ac28bd846c17e7d73f5", - "v1.26.2": "291e85bef77e8440205c873686e9938d7f87c0534e9a491de64e3cc0584295b6", - "v1.26.3": "0f62cbb6fafa109f235a08348d74499a57bb294c2a2e6ee34be1fa83432fec1d", - "v1.26.4": "eea4054825a4c20cc09bc15abcb1354725ad886338e6892141a071caab91d4b6", - "v1.26.5": "c3b3de6a2d7f7e1902c65f6774754e62e86d464ed259509ba29a2a209a515ddf", - "v1.26.6": "8261d35cd374c438104bb5257e6c9dafb8443cd0eed8272b219ec5aa17b8ca40", - "v1.26.7": "71edc4c6838a7332e5f82abb35642ce7f905059a258690b0a585d3ed6de285b3", - "v1.26.8": "e93f836cba409b5ef5341020d9501067a51bf8210cb35649518e5f4d114244cf", - "v1.26.9": "f945c63220b393ddf8df67d87e67ff74b7f56219a670dee38bc597a078588e90", - "v1.26.10": "5752e3908fa1d338eb1fa99a6f39c6a4c27b065cb459da84e35c4ec718879f14", - "v1.26.11": "4a6d2b7204af3cf84cd0e2c670fbb211501050c9a288de49de3c6363d4e0a63e", - "v1.26.12": "b9cfdb6c9a53146330d5694e711c48febb7ec022aea8d16a51b5e12d761580cc", - "v1.27.0": "f8e09630211f2b7c6a8cc38835e7dea94708d401f5c84b23a37c70c604602ddc", - "v1.27.1": "fd3cb8f16e6ed8aee9955b76e3027ac423b6d1cc7356867310d128082e2db916", - "v1.27.2": "1b0966692e398efe71fe59f913eaec44ffd4468cc1acd00bf91c29fa8ff8f578", - "v1.27.3": "7bb7fec4e28e0b50b603d64e47629e812408751bd1e0ce059b2fee83b0e3ff6f", - "v1.27.4": "5178cbb51dcfff286c20bc847d64dd35cd5993b81a2e3609581377a520a6425d", - "v1.27.5": "0158955c59c775165937918f910380ed7b52fca4a26fb41a369734e83aa44874", - "v1.27.6": "7322a6f600de6d0d06cf333bdc24cd2a340bba12920b0c2385c97884c808c810", - "v1.27.7": "61fc334f2c0290270e43fb8a1d4ff07e8cec5642d5a123eb7ab66a134b04ae83", - "v1.27.8": "97ed6739e2803e63fd2d9de78be22d5ba6205bb63179a16ec773063526525a8e", - "v1.27.9": "bda475539fdeda9d8a85a84b967af361af264d0826c121b23b0b62ee9b00cd2d", - "v1.28.0": "f5484bd9cac66b183c653abed30226b561f537d15346c605cc81d98095f1717c", - "v1.28.1": "46954a604b784a8b0dc16754cfc3fa26aabca9fd4ffd109cd028bfba99d492f6", - "v1.28.2": "ea6d89b677a8d9df331a82139bb90d9968131530b94eab26cee561531eff4c53", - "v1.28.3": "06511f03e34d8ee350bd55717845e27ebec3116526db7c60092eeb33a475a337", - "v1.28.4": "edf1e17b41891ec15d59dd3cc62bcd2cdce4b0fd9c2ee058b0967b17534457d7", - "v1.28.5": "f87fe017ae3ccfd93df03bf17edd4089672528107f230563b8c9966909661ef2", - "v1.29.0": "8f7a4bd6bae900a4ddab12bd1399aa652c0d59ea508f39b910e111d248893ff7" - } - }, - "etcd": { - "amd64": { - "v3.4.13": "2ac029e47bab752dacdb7b30032f230f49e2f457cbc32e8f555c2210bb5ff107", - "v3.4.14": "f57141d45ceb727a84e2f0bcd0b98934ea2ad2b59009be90d901710861601651", - "v3.4.15": "3bd00836ea328db89ecba3ed2155293934c0d09e64b53d6c9dfc0a256e724b81", - "v3.4.16": "2e2d5b3572e077e7641193ed07b4929b0eaf0dc2f9463e9b677765528acafb89", - "v3.4.17": "10df17c03fd2f0b3cc3354f73d0b9573af2d3582eff3b210e43e380417b4c644", - "v3.4.18": "495eb67d39d0f595aeb78d438e4634296b24ed6d81a7a364b2e58a242956a3b8", - "v3.4.19": "9ba70e27c17a1faf6d3de89040432189d8071fa27ca156d09d3503989ecd9ccd", - "v3.4.20": "5b3521aeccd0d5fc3df51aaf90bb5d63d0609667ea9863e365531ac733c126ef", - "v3.4.21": "c52ceca477bc78341d181fa71eb4c93a0c34f1756ecccbba50fbaad2e492fdff", - "v3.4.22": "446f9dc03b4d6e61903bf307dd99e5893101de86fc3d969e86b8111a1e59c67e", - "v3.4.23": "f0c53075564c1b98f5e6b1c009dd5c55b911fae1fe4993913246e8d2c8976651", - "v3.4.24": "8b2b35772704e67f956e6bac0fbb739d974155cd19602a00af9dc757a3a8f83d", - "v3.5.0": "864baa0437f8368e0713d44b83afe21dce1fb4ee7dae4ca0f9dd5f0df22d01c4", - "v3.5.1": "728a14914217ce60de2e1299fc1a2c2c5564e7ffd0d9dadf3f5073103ab619b4", - "v3.5.2": "256cad725542d6fd463e81b8a19b86ead4cdfe113f7fb8a1eabc6c32c25d068b", - "v3.5.3": "e13e119ff9b28234561738cd261c2a031eb1c8688079dcf96d8035b3ad19ca58", - "v3.5.4": "b1091166153df1ee0bb29b47fb1943ef0ddf0cd5d07a8fe69827580a08134def", - "v3.5.5": "7910a2fdb1863c80b885d06f6729043bff0540f2006bf6af34674df2636cb906", - "v3.5.6": "4db32e3bc06dd0999e2171f76a87c1cffed8369475ec7aa7abee9023635670fb", - "v3.5.7": "a43119af79c592a874e8f59c4f23832297849d0c479338f9df36e196b86bc396", - "v3.5.8": "d4c1b8d90ad53658f12ffc293afc5694b7bc6cb093af609188649a799e1cc8dc", - "v3.5.9": "d59017044eb776597eca480432081c5bb26f318ad292967029af1f62b588b042", - "v3.5.10": "26e90d024fa2310bc52bb40e7f2132e81640b55f8fc446c00ae07e30af2a44fd" - }, - "arm64": { - "v3.4.13": "1934ebb9f9f6501f706111b78e5e321a7ff8d7792d3d96a76e2d01874e42a300", - "v3.4.14": "5ada2f64ae298e6ed8e608459547705b7deac10d75112bb72aae19138c442f43", - "v3.4.15": "fcc522275300cf90d42377106d47a2e384d1d2083af205cbb7833a79ef5a49d1", - "v3.4.16": "6712ff22aeec39ed19688b1edf6747194193b074548fc62163e4804ca0264512", - "v3.4.17": "3288d52c3b2bff646287a9c4ff72423ce0df04ab7c03e034157a406f9da6ba2a", - "v3.4.18": "32c65d83181d991de8e2f3fac8dcfde922b8ddfe082171a351285900485d36ab", - "v3.4.19": "9304d7e7dc60d7158834478e1c2655a24fc2bff367d3ffda5d17a165b8b99cce", - "v3.4.20": "827e70f2cf3b3873389152d62e785bc9468d3f8ea8c1ec1fd85c9f2a9d65017f", - "v3.4.21": "0b43209e40a1b3194d2ce135d9d9e79ef3798bf009df953bf9c814ac3f69ec6a", - "v3.4.22": "adf302a01decf7273a9e61cf39c8cfe05bfd2c18ffed708a6f668ebde519a8e3", - "v3.4.23": "3f590d7aceb7e28aa4125a09a6021fe05d102349b3bc7afafe1cc981b06b70cd", - "v3.4.24": "9e902a47655370badbdd87c7661444bc7f12c105999ac6971e2f56b1e2d3dd16", - "v3.5.0": "444e10e6880595d75aaf55762901c722049b29d56fef50b2f23464bb7f9db74d", - "v3.5.1": "86203022e23d7368bac23d96095270dc6300f356ea882e435926a9effd7e5f0e", - "v3.5.2": "c068ea3cdcc8ec8fbef75995cef7f63f8206766cdea6b93a91e6d4cdbe79952a", - "v3.5.3": "8b00f2f51568303799368ee4a3c9b9ff8a3dd9f8b7772c4f6589e46bc62f7115", - "v3.5.4": "8e9c2c28ed6b35f36fd94300541da10e1385f335d677afd8efccdcba026f1fa7", - "v3.5.5": "a8d177ae8ecfd1ef025c35ac8c444041d14e67028c1a7b4eda3a69a8dee5f9c3", - "v3.5.6": "888e25c9c94702ac1254c7655709b44bb3711ebaabd3cb05439f3dd1f2b51a87", - "v3.5.7": "1a35314900da7db006b198dd917e923459b462128101736c63a3cda57ecdbf51", - "v3.5.8": "3f4441b293a2d0d4d2f8b2cd9504376e15818f7b865ef4b436e8e6f865f895ff", - "v3.5.9": "bb201c106a61bbab59e2d9f37f4bdff99d50201f513c66b4578741eab581fb28", - "v3.5.10": "ff74a6018d9b2a1320bff30e5a11b4f2f5c2a3d147df8a8bad53c01b9f800ee1" - } - }, - "helm": { - "amd64": { - "v3.2.1": "98c57f2b86493dd36ebaab98990e6d5117510f5efbf21c3344c3bdc91a4f947c", - "v3.2.2": "70996b461a1f99628c243998c8972c72554495b240af360b7adb7482f9486073", - "v3.2.3": "ccc810325c1c34511e7e8098db621556b519a4f230991715bcabe40974772f3a", - "v3.2.4": "7af6d3529def74a4fc8fff85cd03f002c57c43d244a6eb0d41cddbf135879a67", - "v3.3.0": "34025bcbc9f543803aebf54ba3be8dabfb03407b7cc07497e8314ccd592ec973", - "v3.3.1": "96098a3744aa587e3298d82df3ee9fbf9865447b25ed15b037a703d572562f5e", - "v3.3.2": "f8e3032b266008a6f8ab73d8d0476526a05c9fa18b35c661663acd90c7931819", - "v3.3.3": "ce347e004327abf10fba26df70a9a58b9f4d19e81ca39bd14ba483101f8f6262", - "v3.3.4": "5040347643da71cd2e6928a303908804d3673f6c2cc7e9fd5e59c28de25211c4", - "v3.4.0": "58550525963821a227307590627d0c266414e4c56247a5c11559e4abd990b0ae", - "v3.4.1": "7fb909f81b02892f4b062f20edceaf7743484dec9e2acd17a00d5c47df17d005", - "v3.4.2": "d14d54d59558caebe234500f541fc2064b08d725ed8aa76f957f91c8d6a0fc46", - "v3.5.0": "da5b66bec3066e6b51e65ebe85a64d46e4191b80d2fbcaf103d1c4a669442230", - "v3.5.1": "52c8f2238d03c571e0e31de6ab21cb9c4d91c3e576549a91a486dbd3199fd8bd", - "v3.5.2": "018736e597b2e986ff4af2dd51e893fe6af131e36a325ed9b777a478a1cbf17e", - "v3.5.3": "70f4ece831b0de65bdade7d220f75ae6e807085cb3798c1f6c416d2172ce9d60", - "v3.5.4": "ef10138e72714e5f48be2dc0173bfb0e03a2b7d9df60c850544d10690bbe9e8b", - "v3.6.0": "00ce42a9bec0b92863b42d45ede4e5a33848abe17fbf29d26309dac24c813cb6", - "v3.6.1": "3da368031540811398f358a025dc1617db7eec52ef94d07d841e236e784282df", - "v3.6.2": "3e4dd2807e8e00779312343aff59185fbf32971562434920de1ccaf4bae2b746", - "v3.6.3": "6e5498e0fa82ba7b60423b1632dba8681d629e5a4818251478cb53f0b71b3c82", - "v3.7.0": "ee395ce71a0246d196cf20affe8d4abdd66fd0a9e07803add8f3d54f92e247f2", - "v3.7.1": "05156f78d918ca817390c5299b89df55719ccc54eeb24878d3a503cd0c11d3e6", - "v3.7.2": "3a462fb971fbe5e8483f75c2cfd007aeb6443698ef49c90d7d2058206caeccde", - "v3.8.0": "f197057230be75a9db85225c2eaf49f65918bffd5e9e2174a88c41cd5623ea50", - "v3.8.1": "1fe973bb32db1dcacdda9cda150e71e47b517a2af6bfdc8a2735859c8070a0b6", - "v3.8.2": "00886824b190f2e0e2fc0c6523f3e6f50940c39024c1c127feb3c3dafe6386da", - "v3.9.0": "111c4aa64532946feb11a1542e96af730f9748483ee56a06e6b67609ee8cfec3", - "v3.9.1": "067393975bd37dcf3aaaa60eb9a7fc740bc15f680d27dd411ecf122bbe62da24", - "v3.9.2": "24b1c8ee9eca7fbd693c56449a238427840c41a1622242ef049e55e0f42d33a1", - "v3.9.3": "ebfac82c2c7e8425fff1bbfa2b1867502c5703e1e8fe30ee6ec6e52355d51fc1", - "v3.9.4": "d7b04222b47873191178f1afb4338e31f7b1ff0bc47bbb582b49750aed3a4c38", - "v3.10.0": "bc102ba0c9d5fba18b520fbedf63d114e47426a6b6aa0337ecab4a327704d6ab", - "v3.10.1": "8c375d1016b2c9754bad8da3ed07ce33c0f9af3bad8890e61737f5137be7396d", - "v3.10.2": "72f1c0fcfb17b41b89087e9232e50f20c606e44a0edc2bb9737e05d1c75b8c4f", - "v3.10.3": "cc5223b23fd2ccdf4c80eda0acac7a6a5c8cdb81c5b538240e85fe97aa5bc3fb", - "v3.11.0": "930918464763e66f9f99bed076357abc4bc56b040c053fce9884314964ca0fcc", - "v3.11.1": "f5366ce6d2118b0ef7134db08273a058649d88a27deb4b9d3cc0881bb55119a8", - "v3.13.1": "77a68582fac7290df476e1e698f7dd70bb496bf1d475e123664e921f4478c567" - }, - "arm64": { - "v3.2.1": "20bb9d66e74f618cd104ca07e4525a8f2f760dd6d5611f7d59b6ac574624d672", - "v3.2.2": "0a72fa8860f9b63da9120cde4c7b0a1fdd5e10bde5f75dca3ba40ee9663a7641", - "v3.2.3": "4a78939528f747ac3fbe36baf787b5149ff3df25a92c1e191a07fff504f2f1ca", - "v3.2.4": "9f0826e131cc779c52a9a34d776db66697fb866a1829d4bbb0868545e82b144f", - "v3.3.0": "4331fba22d9002647a723ea1b988d5123fdcbe0c7a50a968a7e51017af97f989", - "v3.3.1": "044870af7346d957211c424fb951ff1cfa5cd252f6590efc98c7ec6eb9874b2b", - "v3.3.2": "713927874850ddd92efd309834b7f2743dcda346819d0e33a5a06cfc15307ed0", - "v3.3.3": "38414040f67fc7ca2bb411583efae1a4ffeaef4b067c25afd1005ac1fff63877", - "v3.3.4": "d1f595b6b7175293d1b5a793e1699fb7cc8a09e3e62f1fd40f6e3987069674e7", - "v3.4.0": "89dcc685b7e248d19453381717af4acc1960b14e6d319e6bc39b44bc46fc3aa5", - "v3.4.1": "ebe5d535eaa2188927392a5f3ddb333606dd40bdfa5a4243e91b3e8372d2ccab", - "v3.4.2": "6ff7a9b8517ed4803d7b4ec348179e9f8f7ebdfd1216511efe7dd1e57539b880", - "v3.5.0": "0010b0bfbec92f4446a49c02f9a29204ce2dc637e0b4a5b099c08e6f17e333ed", - "v3.5.1": "f5059851653c0c562f834e11495e6dfed2a376942ed159ecdcb3b6e48db5a795", - "v3.5.2": "0e288ade6dde5611512e176ac7dae76c13d73e5927a5e5a278342b0e0fdb23ae", - "v3.5.3": "cd2836d8d7c51e3e8be5b6df4b2025ead52ddc7c80b0e56323fb74be92826e03", - "v3.5.4": "9d81ef6dff745ed9efc23df6a0e8ce33195819be4ec121523b81b2d7a777ff8f", - "v3.6.0": "373e75da3438c23fa45abbef3bca65649aa72dd8fc48a4d808caa52275a0df86", - "v3.6.1": "5a6b95596d6db0d6b01b362886cf275ae04d70995058571f92a7528051fadea7", - "v3.6.2": "3622821d047e4478045d20b8fc089e9f8004070eec3b0e5a0b12e4cbf7954b8b", - "v3.6.3": "fce1f94dd973379147bb63d8b6190983ad63f3a1b774aad22e54d2a27049414f", - "v3.7.0": "f92d3ef548f22490176e8be3f26525e48be08b13e080138d826169b2619b9a4e", - "v3.7.1": "4f7cafa09cd2c3cffb0c31aa0babb42a722d055c05c60620a422c1da99ecfa49", - "v3.7.2": "bb3d79ec955545edc9fb0728cee9a91e12c9470a41de8dba3a49c54bd7eb03ca", - "v3.8.0": "833ccc455ab0256294c129812f2d77fe6a0d1c47e5628fe315cb9fed1490c2d1", - "v3.8.1": "b2bb3ae4909b4331df19cb9cc5abb71047f03c0e0dbe11a57fa7415a9a63fab3", - "v3.8.2": "c7fedba501405b07cf2d23e8cbe726aa87c6c06adf6a814d85d0a952246f7400", - "v3.9.0": "2fcc6ffdaa280465f5a5c487ca87ad9bdca6101c714d3346ca6adc328e580b93", - "v3.9.1": "a6500b8676a387d70c020a07b001a81930ea4209c296b6307cd6745c844dcf3b", - "v3.9.2": "07b1078e1f4c0084744cde6d74d9bdc5a7c98e1d233e6632a6bee1227b3b0375", - "v3.9.3": "c39be8fa9f253d7c7992a9c2f33d3fc6ae5cf571d88eba1b2e0086b0527df65d", - "v3.9.4": "26b4ba7e1ec5f8de2c171eb615a689889a0110a9c7df7bbcdd39fe4fd8d156ff", - "v3.10.0": "3d37439910b7140cc078eba6c34979321ec3c020db54f30a8a86a0d9e92bac85", - "v3.10.1": "50d56555272e21833508acf470f4e29fdc14bd05f95dd693d8fc26a0ee779d58", - "v3.10.2": "a60d66e54d28529d41bef1309d32f1d6c47ba2201ca4dfdab17a33834c0ffc52", - "v3.10.3": "2b75e7112367ca237cd39408353dcfef9b81b1b5cf78e4826bdccb061dbb8dbd", - "v3.11.0": "f8ab55641ca83ce6d2ab7c048ed86794c9dc15f7e197fb7c6e7c2ec3c2d76c12", - "v3.11.1": "2de8125127d555a006290f7dcd57235b1d83444a020b3bf7248c77cd6d3a629b", - "v3.13.1": "aa8b2996fa70837083286fb3a33a153105ff11067ae2424b1ae7cf250b0dc353" - } - }, - "kubecni": { - "amd64": { - "v0.8.2": "21283754ffb953329388b5a3c52cef7d656d535292bda2d86fcdda604b482f85", - "v0.8.6": "994fbfcdbb2eedcfa87e48d8edb9bb365f4e2747a7e47658482556c12fd9b2f5", - "v0.9.1": "962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7", - "v1.0.0": "5894883eebe3e38f4474810d334b00dc5ec59bd01332d1f92ca4eb142a67d2e8", - "v1.0.1": "5238fbb2767cbf6aae736ad97a7aa29167525dcd405196dfbc064672a730d3cf", - "v1.1.0": "05d46ac19d01669d424ee57401c0deba101763ac494858064b4ea4ffdcc37c5d", - "v1.1.1": "b275772da4026d2161bf8a8b41ed4786754c8a93ebfb6564006d5da7f23831e5", - "v1.2.0": "f3a841324845ca6bf0d4091b4fc7f97e18a623172158b72fc3fdcdb9d42d2d37", - "v1.3.0": "754a71ed60a4bd08726c3af705a7d55ee3df03122b12e389fdba4bea35d7dd7e" - }, - "arm64": { - "v0.8.2": "922b6a3b864a9849cc5bd1672823b1891094260461c2cdd9718778ca8dcaef82", - "v0.8.6": "43fbf750c5eccb10accffeeb092693c32b236fb25d919cf058c91a677822c999", - "v0.9.1": "ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0", - "v1.0.0": "736335bc5923a37cfb6cc2305489ce6206bcc565004f525b5f7c3604f092aa3a", - "v1.0.1": "2d4528c45bdd0a8875f849a75082bc4eafe95cb61f9bcc10a6db38a031f67226", - "v1.1.0": "33fc7b8d9d5be2d7f95e69e6a9e2af206879942f1e6b7615c04017dce5067f1a", - "v1.1.1": "16484966a46b4692028ba32d16afd994e079dc2cc63fbc2191d7bfaf5e11f3dd", - "v1.2.0": "525e2b62ba92a1b6f3dc9612449a84aa61652e680f7ebf4eff579795fe464b57", - "v1.3.0": "de7a666fd6ad83a228086bd55756db62ef335a193d1b143d910b69f079e30598" - } - }, - "k3s": { - "amd64": { - "v1.20.2": "ce3055783cf115ee68fc00bb8d25421d068579ece2fafa4ee1d09f3415aaeabf", - "v1.20.4": "1c7b68b0b7d54f21a9c1727545a7db181668115f161a3986bc137261dd817e98", - "v1.21.4": "47e686ad5390670da79a467ba94399d72e472364bc064a20fecd3937a8d928b5", - "v1.21.6": "89eb5f3d12524d0a9d5b56ba3e2707b106e1731dd0e6d2e7b898ac585f4959df", - "v1.23.13": "334b42a96a65d7e54555827ca31f80896bf18978952f5aa5b8bb83bfdff3db0b", - "v1.24.7": "ec346c909f23b32f9ab7c3ccf5bfa74c89a0515191701ede83556345b70abdca" - }, - "arm64": { - "v1.21.4": "b7f8c026c5346b3e894d731f1dc2490cd7281687549f34c28a849f58c62e3e48", - "v1.21.6": "1f06a2da0e1e8596220a5504291ce69237979ebf520e2458c2d72573945a9c1d", - "v1.23.13": "690a9e278d3e9b4b3884c787770ddda4b4d04490b09732913edaa9e9ede05c88", - "v1.24.7": "191de5ae89cd36a8e3a7582778c9839e90c6685e6ba32d0fb1249039f92a9058" - } - }, - "k8e": { - "amd64": { - "v1.21.14": "e0b7dfcf3da936859e19684b2a847cb4f5cadf4d21c3373140886c5fa997a6b8", - "v1.25.5": "0913de7c371f0386c0e5e99924acab8fa4b18aa6ff4a376c200f9f39cbffead0" - }, - "arm64": { - "v1.21.14": "0f863969df8178b12655e884d3095d850dbe63675b5a334878b2c7478fc9fac1", - "v1.25.5": "d58cf3a0c03b0c0fc01eb5406de65c1c85bff5683c7b6ef9ad99f46616d58525" - } - }, - "docker": { - "amd64": { - "20.10.2": "97017e32a8ecbdd1826bb3c7b1424303ee0dea3f900d33591b1df5e394ed4eed", - "20.10.3": "47065a47f0692cd5af03073c7386fe090d9ef5ac88a7d8455a884d8e15809be5", - "20.10.4": "6ec28b6a251e093f5cf32569c4bfce4821eda02923b33c060694e6ca2c851daa", - "20.10.5": "3f18edc66e1faae607d428349e77f9800bdea554528521f0f6c49fc3f1de6abf", - "20.10.6": "e3b6c3b11518281a51fb0eee73138482b83041e908f01adf8abd3a24b34ea21e", - "20.10.7": "34ad50146fce29b28e5115a1e8510dd5232459c9a4a9f28f65909f92cca314d9", - "20.10.8": "7ea11ecb100fdc085dbfd9ab1ff380e7f99733c890ed815510a5952e5d6dd7e0", - "20.10.9": "caf74e54b58c0b38bb4d96c8f87665f29b684371c9a325562a3904b8c389995e", - "20.10.10": "1719446f99cd56e87d0c67019996af4ea859f11891bfd89de2252d6c916ccaaa", - "20.10.11": "dd6ff72df1edfd61ae55feaa4aadb88634161f0aa06dbaaf291d1be594099ff3", - "20.10.12": "ee9b5be14e54bf92f48c82c2e6a83fbdd1c5329e8f247525a9ed2fe90d9f89a5", - "20.10.13": "39edf7c8d773939ff5e4d318ae565691a9c7e754ed768e172757e58898fb7079", - "20.10.14": "7ca4aeeed86619909ae584ce3405da3766d495f98904ffbd9d859add26b83af5", - "20.10.15": "9ccfc39305ae1d8882d18c9c431544fca82913d6df717409ac2244ac58c4f070", - "20.10.16": "b43ac6c4d2f0b64e445c6564860e4fccd6331f4a61815a60642c7748b53c59ff", - "20.10.17": "969210917b5548621a2b541caf00f86cc6963c6cf0fb13265b9731c3b98974d9", - "20.10.18": "0629b063fa3aa5660f3fb96f67edb0e20e92d5050b82403f95faf1c142177401", - "20.10.19": "ddcd732baaa03958cc8f326a5dca09bcd8f348bb7d2737aaf67bbdd7d80302d1", - "20.10.20": "a303cee9125c89abbbb6c4f044b3e2c01c7895e373b90d8de16a7ed25bb2530a", - "20.10.21": "2582bed8772b283bda9d4565c0af76ee653c93d93dc6b8d0aad795d731a1bb81", - "20.10.22": "945c3a3ddcb79ee7307496c2f39eb3d8372466e8654e63d60bbb462e4a3c1427", - "20.10.23": "0ee39f72cc434137d294c14d30897826bad6e24979e421f51a252769ad37e6d1", - "23.0.0": "6a03bbda96845b7451be2f6aba69c3816c60a97de318e83fd1b39d1be262d8af", - "23.0.1": "ec8a71e79125d3ca76f7cc295f35eea225f4450e0ffe0775f103e2952ff580f6", - "24.0.6": "99792dec613df93169a118b05312a722a63604b868e4c941b1b436abcf3bb70f" - }, - "arm64": { - "20.10.2": "9ea59f249ae92bbaa9831a22f2affa2edc9e824f9daaba831ca51d6d22ef2df5", - "20.10.3": "4dcd105f721297f314bb53622e67dd981a743d72f4b2bfe4f42a8790e0892c82", - "20.10.4": "bd9fb2f770eb508b3273237c5604266b6cd93789d048069bd6b16efbf8919fca", - "20.10.5": "83157b92d7469117c2720fc44074749e080b3c510ae35b8a57c66a016cf07dd5", - "20.10.6": "998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910", - "20.10.7": "be8c9a5a06ebec8fb1d36e867cd00fb5777746a9812a0cae2966778ff899c525", - "20.10.8": "4eb9d5e2adf718cd7ee59f6951715f3113c9c4ee49c75c9efb9747f2c3457b2b", - "20.10.9": "0259f8b6572f02cf0dafd7388ca0e4adfdbbfaba81cfb1b7443e89fccbed22c7", - "20.10.10": "8db47cdcd7ac6e082c9ce83347d8fb99eaa01e04b0c8d94851e8d58f350a3633", - "20.10.11": "87a4219c54552797ffd38790b72832372a90eceb7c8e451c36a682093d57dae6", - "20.10.12": "e1f964e9a7a824bbe2164560c0eb335fab9cc7ee9eb90da36c250c073459cf17", - "20.10.13": "debed306ed9a4e70dcbcb228a0b3898f9730099e324f34bb0e76abbaddf7a6a7", - "20.10.14": "ea971edc1179088bfd25edd04a0c12848143d15cb8202ebb93a6a08973464fd0", - "20.10.15": "46102273fab8d6b8a7cf248a928ebaa4bee43114001c593b0d07092a34a439e1", - "20.10.16": "2f35d8d422b63a59279084c159c9092b63b6d974a7fcd868167aee4cc5f79f3b", - "20.10.17": "249244024b507a6599084522cc73e73993349d13264505b387593f2b2ed603e6", - "20.10.18": "aa2b2da571fb9160df87fd5a831f203fb97655e35fb9c4e8d46e72078ae16acf", - "20.10.19": "a04414b3fcf537f0cff17cf01e2b7cb3e39013c10d12e7959547f11aaf71f63c", - "20.10.20": "ef69a2a8ddb87026a8b19e240b2ae3087764b7285860df7faee24e04024f2eb7", - "20.10.21": "b4ceb6151d4dd1bfc7557f5fe0317e29cfcac91f798c34fae7dee891a811f8ee", - "20.10.22": "2c75cd6c3dc9b81cb5bde664c882e4339a2054e09cf09606f9f7dd6970e7f078", - "20.10.23": "5c40bb7dcd1aad94be49ad75d24e7fd409119ed0eaad04f5d13c4fddfb397c8a", - "23.0.0": "2919ff3448187d4f13cfbe2332707cff3f6dcf2baaac42a34bea8dd21f434f4a", - "23.0.1": "3865f837dbd951b19eeb5f7d87aada2e865b2017e9462fe389f0e5d9a438324d", - "24.0.6": "d9f58aecc42451503e82e6e0562cafa1812b334c92186a7f486e111e70a0f5bd" - } - }, - "containerd": { - "amd64": { - "1.6.2": "3d94f887de5f284b0d6ee61fa17ba413a7d60b4bb27d756a402b713a53685c6a", - "1.6.3": "306b3c77f0b5e28ed10d527edf3d73f56bf0a1fb296075af4483d8516b6975ed", - "1.6.4": "f23c8ac914d748f85df94d3e82d11ca89ca9fe19a220ce61b99a05b070044de0", - "1.6.5": "cf02a2da998bfcf61727c65ede6f53e89052a68190563a1799a7298b0cea86b4", - "1.6.6": "0212869675742081d70600a1afc6cea4388435cc52bf5dc21f4efdcb9a92d2ef", - "1.6.7": "52e817b712d521b193773529ff33626f47507973040c02474a2db95a37da1c37", - "1.6.8": "3a1322c18ee5ff4b9bd5af6b7b30c923a3eab8af1df05554f530ef8e2b24ac5e", - "1.6.9": "9ee2644bfb95b23123f96b564df2035ec94a46f64060ae12322e09a8ec3c2b53", - "1.6.10": "dd1f4730daf728822aea3ba35a440e14b1dfa8f1db97288a59a8666676a13637", - "1.6.11": "21870d7022c52f5f74336d440deffb208ba747b332a88e6369e2aecb69382e48", - "1.6.12": "a56c39795fd0d0ee356b4099a4dfa34689779f61afc858ef84c765c63e983a7d", - "1.6.13": "97f00411587512e62ec762828e581047b23199f8744754706d09976ec24a2736", - "1.6.14": "7da626d46c4edcae1eefe6d48dc6521db3e594a402715afcddc6ac9e67e1bfcd", - "1.6.15": "191bb4f6e4afc237efc5c85b5866b6fdfed731bde12cceaa6017a9c7f8aeda02", - "1.6.16": "2415b431a900275c14942f87f751e1e13d513c1c2f062322b5ca5a9a2190f22a", - "1.6.17": "5f0584d000769d0cf08fc0e25135614ef5bf52971a6069175c78437699f3b8d4", - "1.6.18": "c4e516376a2392520a87abea94baf2045cc3a67e9e0c90c75fb6ed038170561e", - "1.6.19": "3262454d9b3581f4d4da0948f77dde1be51cfc42347a1548bc9ab6870b055815", - "1.6.20": "bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2", - "1.6.21": "04dcc1b99368492caee758583e531392683268197e58156888a3cea2941117b6", - "1.6.22": "5671eb4eba97f0ec98223c84401c9aeb21d0ef16ac3ece3eb8fadd46174d7eab", - "1.6.23": "bcf16bb63a295721a2603e9a56602c5d18e5443df04a9f2c1ca5328f41556fcc", - "1.6.24": "a56fac5ba03c3d6f74ceae14abdc9fafabcba900105e9890c0ac895cc00164ad", - "1.7.0": "b068b05d58025dc9f2fc336674cac0e377a478930f29b48e068f97c783a423f0", - "1.7.1": "9504771bcb816d3b27fab37a6cf76928ee5e95a31eb41510a7d10ae726e01e85", - "1.7.2": "2755c70152ab40856510b4549c2dd530e15f5355eb7bf82868e813c9380e22a7", - "1.7.3": "de7f61aacba88ee647a7dcde1ca77672ec44ab9fb3e58ae90c0efc9b2d8f3068", - "1.7.4": "fc070fabfe3539d46ae5db160d18381270928b3f912e2e800947e9fbd43f510c", - "1.7.5": "33609ae2d5838bc5798306a1ac30d7f2c6a8cff785ca6253d2be8a8b3ccbab25", - "1.7.6": "58408cfa025003e671b0af72183b963363d519543d0d0ba186037e9c57489ffe", - "1.7.7": "371de359d6102c51f6ee2361d08297948d134ce7379e01cb965ceeffa4365fba", - "1.7.8": "5f1d017a5a7359514d6187d6656e88fb2a592d107e6298db7963dbddb9a111d9" - }, - "arm64": { - "1.6.2": "a4b24b3c38a67852daa80f03ec2bc94e31a0f4393477cd7dc1c1a7c2d3eb2a95", - "1.6.3": "354e30d52ff94bd6cd7ceb8259bdf28419296b46cf5585e9492a87fdefcfe8b2", - "1.6.4": "0205bd1907154388dc85b1afeeb550cbb44c470ef4a290cb1daf91501c85cae6", - "1.6.5": "2833e2f0e8f3cb5044566d64121fdd92bbdfe523e9fe912259e936af280da62a", - "1.6.6": "807bf333df331d713708ead66919189d7b142a0cc21ec32debbc988f9069d5eb", - "1.6.7": "4167bf688a0ed08b76b3ac264b90aad7d9dd1424ad9c3911e9416b45e37b0be5", - "1.6.8": "b114e36ecce78cef9d611416c01b784a420928c82766d6df7dc02b10d9da94cd", - "1.6.9": "140197aee930a8bd8a69ff8e0161e56305751be66e899dccd833c27d139f4f47", - "1.6.10": "6d655e80a843f480e1c1cead18479185251581ff2d4a2e2e5eb88ad5b5e3d937", - "1.6.11": "1b34d8ff067da482af021dac325dc4e993d7356c0bd9dc8e5a3bb8271c1532de", - "1.6.12": "0a0133336596b2d1dcafe3587eb91ab302afc28f273614e0e02300694b5457a0", - "1.6.13": "8c7892ae7c2e96a4a9358b1064fb5519a5c0528b715beee67b72e74d7a644064", - "1.6.14": "3ccb61218e60cbba0e1bbe1e5e2bf809ac1ead8eafbbff36c3195d3edd0e4809", - "1.6.15": "d63e4d27c51e33cd10f8b5621c559f09ece8a65fec66d80551b36cac9e61a07d", - "1.6.16": "c2bf51fde02ec9cf8b9c18721bc4f53bd1f19fb2bb3251f41ece61af7347e082", - "1.6.17": "7e110faa738bff2f5f0ffd54c4ec2c17c05fd2af6de4877c839794ca3dadd61c", - "1.6.18": "56b83a0bc955edc5ebaa3bd0f788e654b63395be00fcb1bd03ff4bdfe4b5e1e7", - "1.6.19": "25a0dd6cce4e1058824d6dc277fc01dc45da92539ccb39bb6c8a481c24d2476e", - "1.6.20": "c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce", - "1.6.21": "d713d8fbec491705ffe8c33ecc9051a904f6eedc92574928e1d33616f291c583", - "1.6.22": "7882d6e7f4e97dcba041c37592c4cb9e7a5b4d972380c74d959e388b12d57d01", - "1.6.23": "ea7afb82dc5789307e684ef9b4a55ce1ee9a05dc02c2118df640b01207208c45", - "1.6.24": "1d741e9e2d907f02a8b2a46034a28ff9aacdba88c485cef2f4bad18be9ea23ba", - "1.7.0": "e7e5be2d9c92e076f1e2e15c9f0a6e0609ddb75f7616999b843cba92d01e4da2", - "1.7.1": "1f828dc063e3c24b0840b284c5635b5a11b1197d564c97f9e873b220bab2b41b", - "1.7.2": "d75a4ca53d9addd0b2c50172d168b12957e18b2d8b802db2658f2767f15889a6", - "1.7.3": "85d2eaedabff57ac1d7cd3884bf232155c4c46491f6b071982e4f7b684b74445", - "1.7.4": "ea5a04379bd4252fc1e0b7b37f69cd516350c5269054483535d6eab7a0c79d2e", - "1.7.5": "98fc6990820d52d45b56ea2cda808157d4e61bb30ded96887634644c03025fa9", - "1.7.6": "d844a1c8b993e7e9647f73b9814567004dce1287c0529ce55c50519490eafcce", - "1.7.7": "0a104f487193665d2681fcb5ed83f2baa5f97849fe2661188da835c9d4eaf9e3", - "1.7.8": "3fc551e8f51150804d80cc1958a271bd2252b6334f0355244d0faa5da7fa55d1" - } - }, - "runc": { - "amd64": { - "v1.1.1": "5798c85d2c8b6942247ab8d6830ef362924cd72a8e236e77430c3ab1be15f080", - "v1.1.2": "e0436dfc5d26ca88f00e84cbdab5801dd9829b1e5ded05dcfc162ce5718c32ce", - "v1.1.3": "6e8b24be90fffce6b025d254846da9d2ca6d65125f9139b6354bab0272253d01", - "v1.1.4": "db772be63147a4e747b4fe286c7c16a2edc4a8458bd3092ea46aaee77750e8ce", - "v1.1.9": "b9bfdd4cb27cddbb6172a442df165a80bfc0538a676fbca1a6a6c8f4c6933b43", - "v1.1.10": "81f73a59be3d122ab484d7dfe9ddc81030f595cc59968f61c113a9a38a2c113a" - }, - "arm64": { - "v1.1.1": "20c436a736547309371c7ac2a335f5fe5a42b450120e497d09c8dc3902c28444", - "v1.1.2": "6ebd968d46d00a3886e9a0cae2e0a7b399e110cf5d7b26e63ce23c1d81ea10ef", - "v1.1.3": "00c9ad161a77a01d9dcbd25b1d76fa9822e57d8e4abf26ba8907c98f6bcfcd0f", - "v1.1.4": "dbb71e737eaef454a406ce21fd021bd8f1b35afb7635016745992bbd7c17a223", - "v1.1.9": "b43e9f561e85906f469eef5a7b7992fc586f750f44a0e011da4467e7008c33a0", - "v1.1.10": "4830afd426bdeacbdf9cb8729524aa2ed51790b8c4b28786995925593708f1c8" - } - }, - "crictl": { - "amd64": { - "v1.22.0": "45e0556c42616af60ebe93bf4691056338b3ea0001c0201a6a8ff8b1dbc0652a", - "v1.23.0": "b754f83c80acdc75f93aba191ff269da6be45d0fc2d3f4079704e7d1424f1ca8", - "v1.24.0": "3df4a4306e0554aea4fdc26ecef9eea29a58c8460bebfaca3405799787609880", - "v1.24.1": "4006d3af110b144865be0d308145e1f90f7b40dfa0b208d82d3d1d25f3b7bf97", - "v1.24.2": "53432dd365f7bf045c1e65ee2e44c74df6a0a8f0a798f640a945c8e07ad8682e", - "v1.25.0": "86ab210c007f521ac4cdcbcf0ae3fb2e10923e65f16de83e0e1db191a07f0235", - "v1.26.0": "cda5e2143bf19f6b548110ffba0fe3565e03e8743fadd625fee3d62fc4134eed", - "v1.28.0": "8dc78774f7cbeaf787994d386eec663f0a3cf24de1ea4893598096cb39ef2508" - }, - "arm64": { - "v1.22.0": "a713c37fade0d96a989bc15ebe906e08ef5c8fe5e107c2161b0665e9963b770e", - "v1.23.0": "91094253e77094435027998a99b9b6a67b0baad3327975365f7715a1a3bd9595", - "v1.24.0": "b6fe172738dfa68ca4c71ade53574e859bf61a3e34d21b305587b1ad4ab28d24", - "v1.24.1": "02cc96f03fcd6fbf36746fe9cc29566bf5ddd223b4fcb7e3f948566cee45e6ca", - "v1.24.2": "c1410b15d5009287c94eabcc63b14dba68726771fa87fef743867df2eea0a6d5", - "v1.25.0": "651c939eca010bbf48cc3932516b194028af0893025f9e366127f5b50ad5c4f4", - "v1.26.0": "b632ca705a98edc8ad7806f4279feaff956ac83aa109bba8a85ed81e6b900599", - "v1.28.0": "06e9224e42bc5e23085751e93cccdac89f7930ba6f7a45b8f8fc70ef663c37c4" - } - }, - "registry": { - "amd64": { - "2": "7706e46674fa2cf20f734dfb7e4dd7f1390710e9c0a2c520563e3c55f3e4b5c5" - }, - "arm64": { - "2": "a6e98123b850da5f6476c08b357e504de352a00f656279ec2636625d352abd5a" - } - }, - "compose": { - "amd64": { - "v2.2.2": "92551cd3d22b41536ce8345fe06795ad0d08cb3c17b693ecbfe41176e501bfd4", - "v2.2.3": "f15a7cd54a85f760a8ece92da63fb85b93036c72ead8056391b8a3fe9dc73134", - "v2.3.0": "c3b42086f3b6b4cce2d9ec21a83690960561d0ee41aeb359b810908b56dd0906", - "v2.3.1": "94e5a9327d94c67ccdfd40d0aa226ddf8130aa2a6aaf673f916b4d7fec36cae8", - "v2.3.2": "c999dd4ed26c4549d94e29c86af080e1e1c8490b1586f2bd678abdddeb390a70", - "v2.3.3": "d31e90dda58e21a6463cb918868421b4b58c32504b01b1612d154fe6a9167a91", - "v2.3.4": "bdb11922a5895895ea4be0aea6b9105bf4dd0a4cc3486a753d4afefe2caddbd2", - "v2.4.0": "83527297959403d20a4fd4f00d3db6d9bbbc0cc5704787ec4396cd706a6c8bb4", - "v2.4.1": "6dc9096eb6aed220e9b876b3ae6edbffa8bb3beaa5936d11495acd90ea6246f1", - "v2.5.0": "6296d17268c77a7159f57f04ed26dd2989f909c58cca4d44d1865f28bd27dd67", - "v2.5.1": "d99de1ea7616f2a4c7914d37674f0650660a5e832be555805a71c0fc377233c9", - "v2.6.0": "4eb9084cd9e33d906bd1ea11b5bc2e77a43f8ffbe7228bcf7c829a7687f5c4bb", - "v2.6.1": "ed79398562f3a80a5d8c068fde14b0b12101e80b494aabb2b3533eaa10599e0f", - "v2.7.0": "184df811a70366fa339e99df38fc6ff24fc9e51b3388335efe51c1941377d4ce", - "v2.8.0": "e16838a28c297644908820c2fd8e7750f674603d2f1be7ffc1fc8870fdd3fec7", - "v2.9.0": "3be9ce88ecba41b734e3fc8e59a9b11531133761414a78827d1615aadb5ef1f5", - "v2.10.0": "0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5", - "v2.10.1": "dfe2e3f134cc053e020891a11c23a0923eb49ee35556ec40b37d098eaa5a55f6", - "v2.10.2": "41e9657c8abd7d656c3a40df1ae9c1171930313707a3abd5420ec8852b59eeb7", - "v2.11.0": "e15e0d04bb2ae3885a375a575f27ebd79679f01df2dcb1257bd2c5afe1966122", - "v2.11.1": "f44b91e9ce260c2080dd2929dc2da06068be7a84b730d31522ecd3c3178b1753", - "v2.11.2": "1178848502b0771b96895febeb4b1736acd5019c4ed71a8efbabf6915185fe8a", - "v2.12.0": "2054f3a24cb6814b390bd22c95fa37af7675831e37776bb1473a29a912d48b4b", - "v2.12.1": "e83b45a94d8453b2db550018ebe9fb663ac87abb56c5fb32e7f62717c64f4b20", - "v2.12.2": "36d1728ce001c7f021294be43bdfa3f508038bb00886c34b0794f7731cc9bf4b", - "v2.13.0": "943ff254867e1c23cd6414d7255790daddc7ab69013dd79ba5c172410cbafb14", - "v2.14.0": "fdf634ab2b01aca33372bef2bf866699ef2e1f2dab19972e37967b1fc2a11402", - "v2.14.1": "e0c916bd108c49830c7847e416787387398f97de8dda7c1c35dad59438692664", - "v2.14.2": "d056a8330a01f22c249b9fa03ad0d5be889b79b648cad43c8549eb4c3f8ff0ba", - "v2.15.0": "ba481d45be2b137a2a185abd05f61d6d7766dbedfa038f16e4705760767a206e", - "v2.15.1": "bcfd9ea51dee4c19dccdfaeef0e7956ef68bf14f3d175933742061a7271ef0f5", - "v2.16.0": "54ab01967b05e392e6bf13afbc654146890b9fa40501b40aca83a2db18f10427" - }, - "arm64": { - "v2.2.2": "280cb5ad9299fdc0854fa1ca7c7224d10cdbe84a87678e966f0516f12846dadd", - "v2.2.3": "acdef6cd8ade7ade54037846be2e8291f3a0f735a5c89460a79d8a28747c0f48", - "v2.3.0": "75e486a093000421d88746dbc7906d123e1b407342ed57a902692a512ac00a8f", - "v2.3.1": "c8c160624407fc0757acf2f666f1a8aa86300c8be9afeb37355ac61a2e25f52f", - "v2.3.2": "a01d2e37549b48b61df6f65e3d9f2fb987efe7849b5cd07c8be85c36fec11467", - "v2.3.3": "27a57b26b01a3c5090c390feafb76ec693919c08ef92fe73270baaf65f5b0670", - "v2.3.4": "5c6c936fce3063257bf197b4f7a37f3b96c04b172e39ba6ea7d799f8ccd4055d", - "v2.4.0": "743a8246ae274530433c8a3fe09925900af45a2e24d4f45355959a4b71a1964a", - "v2.4.1": "5296819ee4fd8b5c71e5f110ff238e2ebb330927863fc91f26baa0ddf4bb1d85", - "v2.5.0": "7efc61cc85fe712f14f04a6886d1481c96fe958be265f67482583b4b713b6a22", - "v2.5.1": "002662ed18a22d9d65d3d2c0358008c7c6a3db7dacb8983488130b3954d00e63", - "v2.6.0": "f2bc74dddaa58add7b428b5a764ccd4f048b366f3eb5c80a77ff06fcdc00b3ce", - "v2.6.1": "2890aade218c145827521efb247b5765ac10ac5c46f21dddb2220c03c98d7f83", - "v2.7.0": "bcc79aff65b35581246feca30d53261eddcfc79285868061b31f3ff86d102563", - "v2.8.0": "74170a438753371968e6211a6f13db6e311b248ba1f952b203798a3dcfb69e33", - "v2.9.0": "6d227b060b2bc3dc5f315a07ae4f647f042755691e2da905b1a21e60a8ae3ddf", - "v2.10.0": "81a27fee03789fbe8e534c20c557543bef7362e44d87e7e81af21a440bbb1016", - "v2.10.1": "2598b299693d3ef6087a8faa02e91e29bc2f4515347a0cf4c9ea18967584f153", - "v2.10.2": "dcac4e71c52264a795d0271b36cd6b2ea0ac266931618fab2e9e9a803ed16a4a", - "v2.11.0": "26088a6f1afbb58c78b2d7a7eaa0ec36df59f9f45729843ca1031ec75472b6bc", - "v2.11.1": "ee53340ff5fc9077ebfbf8302806d2e3ce717f762afcdbe601656e1fe9128bbe", - "v2.11.2": "eb65ae4a4dd4de7dc767935c2bc49e3641d265df1273ea8f577cadb6bd9b4ebf", - "v2.12.0": "3708946bf5d78db23502e95a7ef1f716c16060e098b28acbe604a38b3ab3a51a", - "v2.12.1": "efa8630c6fe56d050937e5fa3f9f7821054be94860eb8b2f0ca485f5a84e6474", - "v2.12.2": "a2c9819115df18ada4e6a68be37f6515121984189d379456bdfd53058e07128b", - "v2.13.0": "4a920b7d5fe5011dd466fa87ea3b47ee5f224b5770337188daee359ec606eb20", - "v2.14.0": "0265f45b30f4f0e1d53c1968c590181f64c546129d967460de382c6de38af191", - "v2.14.1": "49924009641e9a60f178b4085c9d308bb726b5ae74eed46c5d1a7dbc8f04948b", - "v2.14.2": "48ef22ecea70b4b197def1c1bfd2e797f7117db5257f6e505e64f03fdc329a7e", - "v2.15.0": "634e397090ca0e857a898d853ab08d7e2f226328b305026c143c68d6ce0686de", - "v2.15.1": "14d31297794868520cb2e61b543bb1c821aaa484af22b397904314ae8227f6a2", - "v2.16.0": "edaf196a0b9ebe749aa1a42a6ce4550d2c6c2620762aa98c36088a9b96fd22ef" - } - }, - "harbor": { - "amd64": { - "v2.4.1": "cfd799c150b59353aefb34835f3a2e859763cb2e91966cd3ffeb1b6ceaa19841", - "v2.4.2": "7ab03fb4b839e6eba5ba4148fbb1e0c3288fd884f79f1e7c02f37e5f2d0244c0", - "v2.4.3": "a595e5e4ca317ae92f5ed7ed3829aa5c71a1953e72c347aead2f99a328d7371b", - "v2.5.0": "e9af7914d70b4b3b78c01e4fdcd47947b13ff9b16569971a2780997b52699c0a", - "v2.5.1": "846c4e8dc57b9b7b27a0ebbbafd4b40f90f7365d1eae25d0a0432360cfa47b74", - "v2.5.2": "b36a5ca653e66c79d0af11d41db003f83dc44d237bdcf51e87968a37d8b47f87", - "v2.5.3": "c536eaf5dcb35a1f2a5b1c4278380bde254a288700aa2ba59c1fd464bf2fcbf1", - "v2.5.4": "942f8898a5f6e1d546553bc78e1ca527a99241bc10d26bb85eb65f179ece04be", - "v2.5.5": "c3918153bb37c352a418a9b02e60393265413ee3458c061eb2fca465ecc74d49", - "v2.5.6": "064b40501eaaa3481161a96a2354c605df866dd8be15fd89c5922d99b499c14a", - "v2.6.0": "d10e21a793f17a1ef7603593c7988cda684f763c5cd67e1f6ae3fca668573d01", - "v2.6.1": "fe877ff36a0d35b01805cb4b65d76ca0ffb85b1b26334e85edf9ed33e054eab3", - "v2.6.2": "b20e6b62465f5a991cdbc89735a711a5836f36d4163b6ca7363a78299f0907a8", - "v2.6.3": "3a9a79bce1667b85d01d00bc110a8709c3e8678213bcc1510544ecedf64532c1", - "v2.6.4": "e4f96a9c2810439c835f31fae642eaed783a3e9858656504408430f138db00ad", - "v2.7.0": "1e1e79d451d04a9c9953934b966e5698362e1262a933d098bd3874529f80fd43", - "v2.7.1": "b86f161f0b6f4c6b294e62797ff20c24a39c918f4d1fd63728864a0461b3cdc7" - } - }, - "calicoctl": { - "amd64": { - "v3.23.2": "3784200cdfc0106c9987df2048d219bb91147f0cc3fa365b36279ac82ea37c7a", - "v3.26.1": "c8f61c1c8e2504410adaff4a7255c65785fe7805eebfd63340ccd3c472aa42cf" - }, - "arm64": { - "v3.23.2": "232b992e6767c68c8c832cc7027a0d9aacb29901a9b5e8871e25baedbbb9c64c", - "v3.26.1": "bba2fbdd6d2998bca144ae12c2675d65c4fbf51c0944d69b1b2f20e08cd14c22" - } - } -} \ No newline at end of file diff --git a/version/version.go b/version/version.go deleted file mode 100644 index 5f4972cb..00000000 --- a/version/version.go +++ /dev/null @@ -1,82 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package version implements version handling code. -package version - -import ( - _ "embed" - "encoding/json" - "fmt" - "runtime" -) - -//go:embed components.json - -// Components is used to store files sha256 -var Components []byte - -var ( - gitMajor string // major version, always numeric - gitMinor string // minor version, numeric possibly followed by "+" - gitVersion string // semantic version, derived by build scripts - gitCommit string // sha1 from git, output of $(git rev-parse HEAD) - gitTreeState string // state of git tree, either "clean" or "dirty" - buildDate string // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ') -) - -// Info exposes information about the version used for the current running code. -type Info struct { - Major string `json:"major,omitempty"` - Minor string `json:"minor,omitempty"` - GitVersion string `json:"gitVersion,omitempty"` - GitCommit string `json:"gitCommit,omitempty"` - GitTreeState string `json:"gitTreeState,omitempty"` - BuildDate string `json:"buildDate,omitempty"` - GoVersion string `json:"goVersion,omitempty"` - Compiler string `json:"compiler,omitempty"` - Platform string `json:"platform,omitempty"` -} - -// Get returns an Info object with all the information about the current running code. -func Get() Info { - return Info{ - Major: gitMajor, - Minor: gitMinor, - GitVersion: gitVersion, - GitCommit: gitCommit, - GitTreeState: gitTreeState, - BuildDate: buildDate, - GoVersion: runtime.Version(), - Compiler: runtime.Compiler, - Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), - } -} - -// String returns info as a human-friendly version string. -func (info Info) String() string { - return info.GitVersion -} - -// ParseFilesSha256 Load files' sha256 from components.json -func ParseFilesSha256(componentsJSON []byte) (map[string]map[string]map[string]string, error) { - m := make(map[string]map[string]map[string]string) - err := json.Unmarshal(componentsJSON, &m) - if err != nil { - return nil, err - } - return m, nil -}