k8s源码编译方式

发表于 | 分类于

目前需要在mips64le上构建1.14版本的k8s

这里需要golang1.12.1以上的版本，首先要搭建golang环境

golang

获取goang1.12.5
这里我将golang1.12.5 mips64el做成了docker image，

1
docker pull liupeng0518/golang-1.12.5-stretch

这里可以将二进制golang包拷贝出来，或者直接在容器里构建，这里为了提高性能，直接在mips物理机上构建，

如果不是在容器里构建，需要配置环境变量

1
2
3
export GOROOT=/home/gowork/golang
export PATH=$PATH:$GOROOT/bin:
export GOPATH=/home/gowork

克隆源代码

1
2
3
cd $GOPATH
git clone https://github.com/kubernetes/kubernetes.git
git checkout v1.14.1

修改源码内容

修改添加编译内容

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
Subject: [PATCH] k8s add mips64le support

---
 src/k8s.io/kubernetes/build/common.sh                         |  9 +++++++++
 src/k8s.io/kubernetes/cluster/clientbin.sh                    |  5 ++++-
 src/k8s.io/kubernetes/hack/lib/golang.sh                      |  8 ++++++++
 src/k8s.io/kubernetes/hack/lib/util.sh                        |  5 ++++-
 src/k8s.io/kubernetes/hack/local-up-cluster.sh                |  5 ++++-
 src/k8s.io/kubernetes/test/images/image-util.sh               |  2 +-
 .../kubernetes/vendor/github.com/google/cadvisor/fs/fs.go     | 10 +++++-----
 .../vendor/github.com/vishvananda/netns/netns_linux.go        |  1 +
 .../vendor/golang.org/x/sys/unix/zsyscall_linux_mips64le.go   | 11 +++++++++++
 9 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/src/k8s.io/kubernetes/build/common.sh b/src/k8s.io/kubernetes/build/common.sh
index 6ace6a0..e5ebf7b 100755
--- a/src/k8s.io/kubernetes/build/common.sh
+++ b/src/k8s.io/kubernetes/build/common.sh
@@ -132,6 +132,14 @@ kube::build::get_docker_wrapped_binaries() {
           kube-scheduler,s390x/busybox
           kube-proxy,k8s.gcr.io/debian-iptables-s390x:${debian_iptables_version}
         );;
+    "mips64le")
+        local targets=(
+          cloud-controller-manager,mips64le/busybox
+          kube-apiserver,mips64le/busybox
+          kube-controller-manager,mips64le/busybox
+          kube-scheduler,mips64le/busybox
+          kube-proxy,k8s.gcr.io/debian-iptables-mips64le:${debian_iptables_version}
+        );;
   esac
 
   echo "${targets[@]}"
@@ -529,6 +537,7 @@ function kube::build::ensure_data_container() {
       --volume /usr/local/go/pkg/linux_arm_cgo
       --volume /usr/local/go/pkg/linux_arm64_cgo
       --volume /usr/local/go/pkg/linux_ppc64le_cgo
+      --volume /usr/local/go/pkg/linux_mips64le_cgo
       --volume /usr/local/go/pkg/darwin_amd64_cgo
       --volume /usr/local/go/pkg/darwin_386_cgo
       --volume /usr/local/go/pkg/windows_amd64_cgo
diff --git a/src/k8s.io/kubernetes/cluster/clientbin.sh b/src/k8s.io/kubernetes/cluster/clientbin.sh
index a403571..3c72b0c 100755
--- a/src/k8s.io/kubernetes/cluster/clientbin.sh
+++ b/src/k8s.io/kubernetes/cluster/clientbin.sh
@@ -59,8 +59,11 @@ case "$(uname -m)" in
   ppc64le*)
     host_arch=ppc64le
     ;;
+  mips64*)
+    host_arch=mips64le
+    ;;
   *)
-    echo "Unsupported host arch. Must be x86_64, 386, arm, s390x or ppc64le." >&2
+    echo "Unsupported host arch. Must be x86_64, 386, arm, s390x, mips64le or ppc64le." >&2
     exit 1
     ;;
 esac
diff --git a/src/k8s.io/kubernetes/hack/lib/golang.sh b/src/k8s.io/kubernetes/hack/lib/golang.sh
index 01501e7..76054d2 100755
--- a/src/k8s.io/kubernetes/hack/lib/golang.sh
+++ b/src/k8s.io/kubernetes/hack/lib/golang.sh
@@ -107,6 +107,7 @@ else
     linux/arm64
     linux/s390x
     linux/ppc64le
+    linux/mips64le
   )
 
   # The node platforms we build for
@@ -116,6 +117,7 @@ else
     linux/arm64
     linux/s390x
     linux/ppc64le
+    linux/mips64le
     windows/amd64
   )
 
@@ -127,6 +129,7 @@ else
     linux/arm64
     linux/s390x
     linux/ppc64le
+    linux/mips64le
     darwin/amd64
     darwin/386
     windows/amd64
@@ -140,6 +143,7 @@ else
     linux/arm64
     linux/s390x
     linux/ppc64le
+    linux/mips64le
     darwin/amd64
     windows/amd64
   )
@@ -316,6 +320,10 @@ kube::golang::set_platform_envs() {
         export CGO_ENABLED=1
         export CC=s390x-linux-gnu-gcc
         ;;
+      "linux/mips64le")
+        export CGO_ENABLED=1
+        export CC=mips64el-linux-gnu-gcc
+        ;;
     esac
   fi
 }
diff --git a/src/k8s.io/kubernetes/hack/lib/util.sh b/src/k8s.io/kubernetes/hack/lib/util.sh
index 19220c9..ffa7a75 100755
--- a/src/k8s.io/kubernetes/hack/lib/util.sh
+++ b/src/k8s.io/kubernetes/hack/lib/util.sh
@@ -145,8 +145,11 @@ kube::util::host_platform() {
     ppc64le*)
       host_arch=ppc64le
       ;;
+    mips64*)
+      host_arch=mips64le
+      ;;
     *)
-      kube::log::error "Unsupported host arch. Must be x86_64, 386, arm, arm64, s390x or ppc64le."
+      kube::log::error "Unsupported host arch. Must be x86_64, 386, arm, arm64, s390x,mips64le or ppc64le."
       exit 1
       ;;
   esac
diff --git a/src/k8s.io/kubernetes/hack/local-up-cluster.sh b/src/k8s.io/kubernetes/hack/local-up-cluster.sh
index bcf988b..cd26c6f 100755
--- a/src/k8s.io/kubernetes/hack/local-up-cluster.sh
+++ b/src/k8s.io/kubernetes/hack/local-up-cluster.sh
@@ -316,8 +316,11 @@ function detect_binary {
       ppc64le*)
         host_arch=ppc64le
         ;;
+      mips64*)
+        host_arch=mips64le
+        ;;
       *)
-        echo "Unsupported host arch. Must be x86_64, 386, arm, arm64, s390x or ppc64le." >&2
+        echo "Unsupported host arch. Must be x86_64, 386, arm, arm64, s390x mips64le or ppc64le." >&2
         exit 1
         ;;
     esac
diff --git a/src/k8s.io/kubernetes/test/images/image-util.sh b/src/k8s.io/kubernetes/test/images/image-util.sh
index dcfd361..6ccafe1 100755
--- a/src/k8s.io/kubernetes/test/images/image-util.sh
+++ b/src/k8s.io/kubernetes/test/images/image-util.sh
@@ -25,7 +25,7 @@ KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE}")/../.." && pwd -P)"
 source "${KUBE_ROOT}/hack/lib/util.sh"
 
 # Mapping of go ARCH to actual architectures shipped part of multiarch/qemu-user-static project
-declare -A QEMUARCHS=( ["amd64"]="x86_64" ["arm"]="arm" ["arm64"]="aarch64" ["ppc64le"]="ppc64le" ["s390x"]="s390x" )
+declare -A QEMUARCHS=( ["amd64"]="x86_64" ["arm"]="arm" ["arm64"]="aarch64" ["ppc64le"]="ppc64le" ["s390x"]="s390x" ["mips64le"]="mips64")
 
 # Returns list of all supported architectures from BASEIMAGE file
 listArchs() {
diff --git a/src/k8s.io/kubernetes/vendor/github.com/google/cadvisor/fs/fs.go b/src/k8s.io/kubernetes/vendor/github.com/google/cadvisor/fs/fs.go
index 7d0ead9..42a6a02 100644
--- a/src/k8s.io/kubernetes/vendor/github.com/google/cadvisor/fs/fs.go
+++ b/src/k8s.io/kubernetes/vendor/github.com/google/cadvisor/fs/fs.go
@@ -524,8 +524,8 @@ func (self *RealFsInfo) GetDirFsDevice(dir string) (*DeviceInfo, error) {
 		return nil, fmt.Errorf("stat failed on %s with error: %s", dir, err)
 	}
 
-	major := major(buf.Dev)
-	minor := minor(buf.Dev)
+	major := major(uint64(buf.Dev))
+	minor := minor(uint64(buf.Dev))
 	for device, partition := range self.partitions {
 		if partition.major == major && partition.minor == minor {
 			return &DeviceInfo{device, major, minor}, nil
@@ -771,10 +771,10 @@ func getBtrfsMajorMinorIds(mount *mount.Info) (int, int, error) {
 			return 0, 0, err
 		}
 
-		glog.V(4).Infof("btrfs dev major:minor %d:%d\n", int(major(buf.Dev)), int(minor(buf.Dev)))
-		glog.V(4).Infof("btrfs rdev major:minor %d:%d\n", int(major(buf.Rdev)), int(minor(buf.Rdev)))
+		glog.V(4).Infof("btrfs dev major:minor %d:%d\n", int(major(uint64(buf.Dev))), int(minor(uint64(buf.Dev))))
+		glog.V(4).Infof("btrfs rdev major:minor %d:%d\n", int(major(uint64(buf.Rdev))), int(minor(uint64(buf.Rdev))))
 
-		return int(major(buf.Dev)), int(minor(buf.Dev)), nil
+		return int(major(uint64(buf.Dev))), int(minor(uint64(buf.Dev))), nil
 	} else {
 		return 0, 0, fmt.Errorf("%s is not a block device", mount.Source)
 	}
diff --git a/src/k8s.io/kubernetes/vendor/github.com/vishvananda/netns/netns_linux.go b/src/k8s.io/kubernetes/vendor/github.com/vishvananda/netns/netns_linux.go
index 34af6e2..691ba2e 100644
--- a/src/k8s.io/kubernetes/vendor/github.com/vishvananda/netns/netns_linux.go
+++ b/src/k8s.io/kubernetes/vendor/github.com/vishvananda/netns/netns_linux.go
@@ -24,6 +24,7 @@ var SYS_SETNS = map[string]uintptr{
 	"ppc64":   350,
 	"ppc64le": 350,
 	"s390x":   339,
+	"mips64le":   5303,
 }[runtime.GOARCH]
 
 // Deprecated: use syscall pkg instead (go >= 1.5 needed).
diff --git a/src/k8s.io/kubernetes/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64le.go b/src/k8s.io/kubernetes/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64le.go
index f302670..0d14130 100644
--- a/src/k8s.io/kubernetes/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64le.go
+++ b/src/k8s.io/kubernetes/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64le.go
@@ -1608,6 +1608,17 @@ func Getuid() (uid int) {
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func InotifyInit() (fd int, err error) {
+	r0, _, e1 := RawSyscall(SYS_INOTIFY_INIT, 0, 0, 0)
+	fd = int(r0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func Lchown(path string, uid int, gid int) (err error) {
 	var _p0 *byte
 	_p0, err = BytePtrFromString(path)
-- 
2.1.0

对于1.13/1.14版本之后无需修改

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
diff --git a/src/k8s.io/kubernetes/build/common.sh b/src/k8s.io/kubernetes/build/common.sh
index 6ace6a0..e5ebf7b 100755
--- a/src/k8s.io/kubernetes/build/common.sh
+++ b/src/k8s.io/kubernetes/build/common.sh
@@ -132,6 +132,14 @@ kube::build::get_docker_wrapped_binaries() {
           kube-scheduler,s390x/busybox
           kube-proxy,k8s.gcr.io/debian-iptables-s390x:${debian_iptables_version}
         );;
+    "mips64le")
+        local targets=(
+          cloud-controller-manager,mips64le/busybox
+          kube-apiserver,mips64le/busybox
+          kube-controller-manager,mips64le/busybox
+          kube-scheduler,mips64le/busybox
+          kube-proxy,k8s.gcr.io/debian-iptables-mips64le:${debian_iptables_version}
+        );;
   esac

编译

执行make

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
[root@node03 kubernetes]# make
+++ [0515 08:30:35] Building go targets for linux/mips64le:
    ./vendor/k8s.io/code-generator/cmd/deepcopy-gen
+++ [0515 08:31:12] Building go targets for linux/mips64le:
    ./vendor/k8s.io/code-generator/cmd/defaulter-gen
+++ [0515 08:31:40] Building go targets for linux/mips64le:
    ./vendor/k8s.io/code-generator/cmd/conversion-gen
+++ [0515 08:32:51] Building go targets for linux/mips64le:
    ./vendor/k8s.io/kube-openapi/cmd/openapi-gen
+++ [0515 08:33:35] Building go targets for linux/mips64le:
    ./vendor/github.com/jteeuwen/go-bindata/go-bindata
+++ [0515 08:33:40] Building go targets for linux/mips64le:
    cmd/kube-proxy
    cmd/kube-apiserver
    cmd/kube-controller-manager
    cmd/cloud-controller-manager
    cmd/kubelet
    cmd/kubeadm
    cmd/hyperkube
    cmd/kube-scheduler
    vendor/k8s.io/apiextensions-apiserver
    cluster/gce/gci/mounter
    cmd/kubectl
    cmd/gendocs
    cmd/genkubedocs
    cmd/genman
    cmd/genyaml
    cmd/genswaggertypedocs
    cmd/linkcheck
    vendor/github.com/onsi/ginkgo/ginkgo
    test/e2e/e2e.test
    cmd/kubemark
    vendor/github.com/onsi/ginkgo/ginkgo
    test/e2e_node/e2e_node.test

查看编译的二进制文件

1
2
3
4
[root@node03 bin]# ls
apiextensions-apiserver   deepcopy-gen   e2e.test     genman              ginkgo      hyperkube       kube-controller-manager  kubemark        linkcheck
cloud-controller-manager  defaulter-gen  gendocs      genswaggertypedocs  go2make     kubeadm         kubectl                  kube-proxy      mounter
conversion-gen            e2e_node.test  genkubedocs  genyaml             go-bindata  kube-apiserver  kubelet                  kube-scheduler  openapi-gen

编译UOS 20-mips64 hyperkube

https://github.com/kubernetes/kubernetes/blob/master/vendor/github.com/google/cadvisor/machine/machine.go

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
func GetClockSpeed(procInfo []byte) (uint64, error) {
	// s390/s390x, aarch64 and arm32 changes
	if isMips64() || isSystemZ() || isAArch64() || isArm32() {
		return 0, nil
	}

	// First look through sys to find a max supported cpu frequency.
	if utils.FileExists(maxFreqFile) {
		val, err := ioutil.ReadFile(maxFreqFile)
		if err != nil {
			return 0, err
		}
		var maxFreq uint64
		n, err := fmt.Sscanf(string(val), "%d", &maxFreq)
		if err != nil || n != 1 {
			return 0, fmt.Errorf("could not parse frequency %q", val)
		}
		return maxFreq, nil
	}
	// Fall back to /proc/cpuinfo
	matches := cpuClockSpeedMHz.FindSubmatch(procInfo)
	if len(matches) != 2 {
		return 0, fmt.Errorf("could not detect clock speed from output: %q", string(procInfo))
	}
	
	speed, err := strconv.ParseFloat(string(matches[1]), 64)
	if err != nil {
		return 0, err
	}
	// Convert to kHz
	return uint64(speed * 1000), nil
}
...
// mipsx changes
func isMips64() bool {
	arch, err := getMachineArch()
	if err == nil {
		return strings.Contains(arch, "mips64")
	}
	return false
}

这里已经提交pr：https://github.com/google/cadvisor/pull/2376