mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 11:23:50 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0711870f43 | ||
|
|
109dc7fdf6 | ||
|
|
fc54afdba9 | ||
|
|
0623bf582e | ||
|
|
98dc1e5c15 | ||
|
|
049f040c3c | ||
|
|
124b97f6d7 | ||
|
|
1e27d36576 | ||
|
|
f9383aa726 | ||
|
|
b7de4a967e | ||
|
|
b9d8a837af | ||
|
|
5db25a8f2a | ||
|
|
f8d3ff466d | ||
|
|
a522507f95 | ||
|
|
b9933d5589 | ||
|
|
36daa7709d | ||
|
|
0f522da367 | ||
|
|
ea271600ec | ||
|
|
0452ae6a6c | ||
|
|
ec46a34f33 | ||
|
|
18d92dbd45 | ||
|
|
93d15488e4 | ||
|
|
595db62fe7 | ||
|
|
8ef802c259 | ||
|
|
5b106630aa | ||
|
|
1cdbc8beb5 | ||
|
|
d3f89af079 | ||
|
|
707abd5b2d | ||
|
|
29632d5a34 | ||
|
|
e9584d9661 | ||
|
|
6fa3d0cc46 | ||
|
|
c3487d3da7 | ||
|
|
ba0db28dbb | ||
|
|
8e81619d02 | ||
|
|
620be2be16 | ||
|
|
56c94cc08e | ||
|
|
3de1e19780 | ||
|
|
5e9039d728 | ||
|
|
e00a12b099 | ||
|
|
aacdcc4cad | ||
|
|
301b678147 | ||
|
|
47fd5d3fa1 | ||
|
|
7ae51d1ec6 | ||
|
|
190afcc3fd | ||
|
|
faa6167b6b | ||
|
|
b6d7556dda | ||
|
|
ea15476b06 | ||
|
|
53c7161f27 | ||
|
|
30200cb6af | ||
|
|
b5d99110d1 | ||
|
|
6b93148f9c | ||
|
|
d1357c1a3a | ||
|
|
1ffb1e696e | ||
|
|
640c53cda8 | ||
|
|
cc570a4477 | ||
|
|
5c8f1467a1 | ||
|
|
973b8ff067 | ||
|
|
e31fdbc89b | ||
|
|
31922b2bf2 | ||
|
|
b90d92fca9 | ||
|
|
b463ca1a2f | ||
|
|
023155bea5 | ||
|
|
cdadf2f43d | ||
|
|
d0045c643e | ||
|
|
73c6de7de0 | ||
|
|
5fce582bb7 | ||
|
|
a0114c0a72 | ||
|
|
960902735f | ||
|
|
137ce1e9ac | ||
|
|
b2a171632b | ||
|
|
23f3aea4cf | ||
|
|
901e8157de | ||
|
|
9d230521a4 | ||
|
|
10d5b4b32b | ||
|
|
3fc261d27c | ||
|
|
06748f2771 | ||
|
|
5d87ad72d8 | ||
|
|
824f7ad9e1 | ||
|
|
277976bd76 | ||
|
|
d949a238b8 | ||
|
|
cf90f76a35 | ||
|
|
a752f6220e | ||
|
|
37e5684b40 | ||
|
|
6f7ef8711a | ||
|
|
ad3a3c8782 | ||
|
|
c4fc007630 | ||
|
|
c202d4c1b0 | ||
|
|
f9e3e7d1c4 | ||
|
|
896a85d6e4 | ||
|
|
e0064ae097 | ||
|
|
d65bdeef08 | ||
|
|
92658b05a8 | ||
|
|
a635973847 | ||
|
|
0fc242b084 | ||
|
|
f6d391c9da | ||
|
|
150ce9f3f6 | ||
|
|
b51880ed3b | ||
|
|
b64411bda8 | ||
|
|
bb880d6253 | ||
|
|
21c0587900 | ||
|
|
c8645fd232 | ||
|
|
c30f6abb11 | ||
|
|
12914af4d8 | ||
|
|
3429cbe3fc | ||
|
|
d5239ab983 | ||
|
|
bd2dc6d641 | ||
|
|
dfbd8efd26 | ||
|
|
f2dde2b1da | ||
|
|
8efe1db01a | ||
|
|
a8d7615eec | ||
|
|
c4363c4b5e | ||
|
|
a36d7b77be | ||
|
|
85a627b05c | ||
|
|
72d8a9f9a8 | ||
|
|
e22335ba78 | ||
|
|
d54eb9966f | ||
|
|
d734fff322 | ||
|
|
dfaa602661 | ||
|
|
19652c1b83 | ||
|
|
b6a1b84a00 | ||
|
|
6d2000e8da | ||
|
|
c9f573eed8 | ||
|
|
2c92a9ff74 | ||
|
|
60fa26ef45 | ||
|
|
ba83dbda6f | ||
|
|
58d18b68d8 | ||
|
|
f8ef25099c | ||
|
|
0baf3d38c2 | ||
|
|
1f5376c915 | ||
|
|
eaa72cf47f | ||
|
|
e646d92c09 | ||
|
|
3801c5269d | ||
|
|
db5588cac7 | ||
|
|
b52d57eb29 | ||
|
|
a17d61d5e3 | ||
|
|
04fddff3ee | ||
|
|
26d9f898af | ||
|
|
cb8f0ab3a1 | ||
|
|
386d4bfb35 | ||
|
|
ce55922884 | ||
|
|
721c6197f9 | ||
|
|
b834027c5a | ||
|
|
d8ca7d34fe | ||
|
|
22064c3425 | ||
|
|
b9112747b5 | ||
|
|
6690236754 | ||
|
|
4af8c427eb | ||
|
|
4957d8eec6 | ||
|
|
b1501f61dd | ||
|
|
ddd6bee970 | ||
|
|
2c6b68b40e | ||
|
|
b64674018a | ||
|
|
3cb914f7e1 | ||
|
|
ebef579ebb | ||
|
|
576c113c59 | ||
|
|
4e3e2b1b82 | ||
|
|
205ecb5d03 | ||
|
|
fc4208d128 | ||
|
|
48db56ddad | ||
|
|
10d545060f | ||
|
|
41af5a6ee5 | ||
|
|
64e75a286e | ||
|
|
d14d029c73 | ||
|
|
406a80da4b | ||
|
|
2bfc8970d2 | ||
|
|
9b084d4c88 | ||
|
|
eb7cf3de81 | ||
|
|
4260804613 | ||
|
|
d653c5f811 | ||
|
|
03e22ddb43 | ||
|
|
31d2f77ceb | ||
|
|
7de112943b | ||
|
|
7e52b1e195 | ||
|
|
449582343f | ||
|
|
695e3a909c | ||
|
|
cca45b02a2 | ||
|
|
9678fc2106 | ||
|
|
5ff33eb558 | ||
|
|
b324a6536c | ||
|
|
c07ab9c060 | ||
|
|
f4f6855c61 | ||
|
|
3c3682fcce | ||
|
|
01a25755c5 | ||
|
|
bc629665de | ||
|
|
7c277f36ec | ||
|
|
0c90ea9c02 | ||
|
|
c53dad5026 | ||
|
|
6dab56df0d | ||
|
|
409f69c6cf | ||
|
|
99ad7e6021 | ||
|
|
6e66fe2e37 | ||
|
|
b848853add | ||
|
|
9b2b7d4f5a | ||
|
|
916673ae24 | ||
|
|
5307cadb32 | ||
|
|
5b38f22e6e | ||
|
|
131d389fc4 | ||
|
|
f4abefc109 | ||
|
|
39340f7e42 | ||
|
|
7c6324b114 | ||
|
|
bf844d8e46 | ||
|
|
3621987727 | ||
|
|
8c5759a06d | ||
|
|
70777d9b9f | ||
|
|
cb6cf331ca | ||
|
|
7f56a40553 | ||
|
|
a28b668647 | ||
|
|
228b133afa | ||
|
|
853880bd83 | ||
|
|
a112cbe44c | ||
|
|
5afc20dbd3 | ||
|
|
d1bb23645e | ||
|
|
c37d6fc01a | ||
|
|
8c3040db81 | ||
|
|
90d55cd179 | ||
|
|
86a5b6183d | ||
|
|
04300dc2ac | ||
|
|
466bbe4cdd | ||
|
|
82c2ce74db | ||
|
|
10efdc7aab | ||
|
|
803ca3c958 | ||
|
|
9c76896e2d | ||
|
|
bcf35876d1 | ||
|
|
97d69d5336 | ||
|
|
a73e177ecf | ||
|
|
4a4ef3cc3c | ||
|
|
3d8b8ffced | ||
|
|
870a30665b | ||
|
|
ef78631a7c | ||
|
|
9c944377dc | ||
|
|
a54f30b6c6 | ||
|
|
7eb121d5ae | ||
|
|
77098de69a | ||
|
|
de5263364b | ||
|
|
e583dbdd2a | ||
|
|
d81db3c703 | ||
|
|
ade94b0d0a | ||
|
|
ae6292f9f0 | ||
|
|
e16dda88e4 | ||
|
|
105702ebe0 | ||
|
|
903a06a160 | ||
|
|
570b488572 | ||
|
|
4966a3abc7 | ||
|
|
42b72e4006 | ||
|
|
99b7f8fa4b | ||
|
|
85fe01eefa | ||
|
|
75fe6e4ab2 | ||
|
|
ac939ccb89 | ||
|
|
00041f6e08 | ||
|
|
b7a887fea1 | ||
|
|
089446f5b1 | ||
|
|
cc0c8c5f81 | ||
|
|
3741779144 | ||
|
|
580a7fa685 | ||
|
|
87b70a6809 | ||
|
|
27c7611ebf | ||
|
|
9920d65bc0 | ||
|
|
c839ce1b19 | ||
|
|
dc4037925d | ||
|
|
3f3a1341d8 | ||
|
|
301b49b63f | ||
|
|
57a46f46a0 | ||
|
|
8dae81c5ed | ||
|
|
e0f4366f4c | ||
|
|
5df105b1f9 | ||
|
|
f5156cf3a8 | ||
|
|
e90ab4ac60 | ||
|
|
4f7a1f67cd | ||
|
|
913a5d0dd4 | ||
|
|
6f483a4d36 | ||
|
|
7394f7feee | ||
|
|
dbf0de4ce1 | ||
|
|
2930263dfd | ||
|
|
7b424a54dc | ||
|
|
f7e4359b59 | ||
|
|
f7aeb06544 | ||
|
|
c57c79a0ab | ||
|
|
10414fd81c | ||
|
|
2e78a522ab | ||
|
|
762dc219a6 | ||
|
|
21a933c932 | ||
|
|
f0b4a7659a | ||
|
|
cda8cc22bc | ||
|
|
bf88006037 | ||
|
|
9fce75607d | ||
|
|
911475526c | ||
|
|
7eb15b1969 | ||
|
|
676e27c589 | ||
|
|
601ba5fb68 | ||
|
|
57e7582c36 | ||
|
|
7ea2c24c5d | ||
|
|
741a013689 | ||
|
|
ebc1b478b1 | ||
|
|
66a1d6b50f | ||
|
|
6ec81f09f7 | ||
|
|
622521c247 | ||
|
|
6d4846a29a | ||
|
|
d9991aeaef | ||
|
|
9076a0051c | ||
|
|
f875031f06 | ||
|
|
bdba3da2e4 | ||
|
|
f4b52d4c52 | ||
|
|
b4131a3a72 | ||
|
|
f6b92b8561 | ||
|
|
3c8e95eeb3 | ||
|
|
8bac903d29 | ||
|
|
74b53729e1 | ||
|
|
7807f6641a | ||
|
|
c4e862e908 | ||
|
|
81e7e6ceb3 | ||
|
|
63fbf281c7 | ||
|
|
3a75d7f7aa | ||
|
|
fef9c6a520 | ||
|
|
2762154130 | ||
|
|
fb5808e0c3 | ||
|
|
5790d01c6f | ||
|
|
8b3429858d | ||
|
|
2ef6ab998c | ||
|
|
f49a9297c2 | ||
|
|
388f82f322 | ||
|
|
77e4b1376e | ||
|
|
3f758820c1 | ||
|
|
c7892bc7c4 | ||
|
|
f7e0a65e75 | ||
|
|
649555b223 | ||
|
|
973f6dd162 | ||
|
|
65ff7198fe | ||
|
|
28b862f45f | ||
|
|
3bf8e772f8 | ||
|
|
f8fa430257 | ||
|
|
4081d50607 | ||
|
|
409f39390d | ||
|
|
6c8822f269 | ||
|
|
a3553da7f7 | ||
|
|
d49d0a9fc2 | ||
|
|
689b4ecdcc | ||
|
|
e3f40d538d | ||
|
|
9f50b310a0 | ||
|
|
11f99836c3 | ||
|
|
fdd8c5d5e0 | ||
|
|
aac45f3e89 | ||
|
|
58dd880322 | ||
|
|
b6f99c4d83 | ||
|
|
5702727db5 | ||
|
|
4d72a1ce6e | ||
|
|
3172c33fa6 | ||
|
|
42c6e52513 | ||
|
|
4573c99ae3 | ||
|
|
4ffbda1c43 | ||
|
|
2b580a7566 | ||
|
|
eaeb141b09 | ||
|
|
e259052a70 | ||
|
|
7fcfaf7bc9 | ||
|
|
400f0c3e5d | ||
|
|
fa4d0093e1 | ||
|
|
93261f5199 | ||
|
|
9c588d4010 | ||
|
|
96caf21d09 | ||
|
|
55f90302a7 | ||
|
|
21106002be | ||
|
|
8affcf070c | ||
|
|
3723eb09c0 | ||
|
|
4d8bbd559a | ||
|
|
373736b984 | ||
|
|
31faa6d43d | ||
|
|
cb476a53ff | ||
|
|
7c029b2183 | ||
|
|
e88392b50f | ||
|
|
7457c746f0 | ||
|
|
789d9dbfe1 | ||
|
|
6199d9b923 | ||
|
|
d8c54b527e | ||
|
|
4d4cd0948d | ||
|
|
53b601616c | ||
|
|
710d2a6f16 | ||
|
|
986daecbfe | ||
|
|
ae93c966d9 | ||
|
|
670cb759f8 | ||
|
|
662ec97602 | ||
|
|
9b006250b2 | ||
|
|
b0633716b7 | ||
|
|
33a5225302 | ||
|
|
1d8645b262 | ||
|
|
16709e8ed3 | ||
|
|
02a7df86f3 | ||
|
|
64af8c8e53 | ||
|
|
aec57b8d6e | ||
|
|
da167e3134 | ||
|
|
0854171d22 | ||
|
|
d8e69df295 | ||
|
|
0aeec04c31 | ||
|
|
928d29af9e | ||
|
|
d3d52b1818 | ||
|
|
358ccb5e0e | ||
|
|
67814a5c79 | ||
|
|
3afda0c89c | ||
|
|
1a4bf0dcb5 | ||
|
|
02b38f894b | ||
|
|
0e9478488d | ||
|
|
8a880a139d | ||
|
|
e134570a9c | ||
|
|
84ec68e11a | ||
|
|
b278bb24d3 | ||
|
|
4d08393b7c | ||
|
|
28a1f42962 | ||
|
|
22181dd018 | ||
|
|
bd13a7968f | ||
|
|
88945d9954 | ||
|
|
fdd85b34f9 | ||
|
|
6e9d39ac72 | ||
|
|
90db4b5a44 | ||
|
|
8a61c124c7 | ||
|
|
fc8035d672 | ||
|
|
3ffa4ba91d | ||
|
|
2a5629dd28 | ||
|
|
df9cc31a0a | ||
|
|
0a5d3c3cf6 | ||
|
|
bd5c9904be | ||
|
|
e26d1cf4ef | ||
|
|
25b9062152 | ||
|
|
fc65122766 | ||
|
|
ab50f21ad9 | ||
|
|
f0e0073b84 | ||
|
|
2f85e31f5c | ||
|
|
0e441b48a6 | ||
|
|
ed5b08c284 | ||
|
|
5209ebbeef | ||
|
|
d764cefb18 | ||
|
|
7235410b57 | ||
|
|
6f1a0b6114 | ||
|
|
b59bc607bf | ||
|
|
78cbd8002f | ||
|
|
cd5c7ad052 | ||
|
|
634bc97840 | ||
|
|
3c75479e2b | ||
|
|
c0d92f61a1 | ||
|
|
1d29f67c02 | ||
|
|
a98f6d66a3 | ||
|
|
036566629a | ||
|
|
303bd067b5 | ||
|
|
5fbe437ee9 | ||
|
|
03c6f978f4 | ||
|
|
69e4725812 | ||
|
|
1c2c7a3a65 | ||
|
|
afef014832 | ||
|
|
0bb3ba17ae | ||
|
|
1a4a36d510 | ||
|
|
cb096d1f53 | ||
|
|
0423d50106 | ||
|
|
0f50fcb96a | ||
|
|
91d742eba2 | ||
|
|
1d05e7fcbd | ||
|
|
c15b356b81 | ||
|
|
39c6d53a1b | ||
|
|
ef84609010 | ||
|
|
b67fe6223b | ||
|
|
b09dab3477 | ||
|
|
13b9a52f80 | ||
|
|
8a2edd5673 | ||
|
|
3bafa24ea1 | ||
|
|
f6de0e2b93 | ||
|
|
a0e2e6e4ed | ||
|
|
5968dd5d4e | ||
|
|
be0ec7bc2f | ||
|
|
1f161f53dc | ||
|
|
26dbc6c905 | ||
|
|
2f0dda384d | ||
|
|
5c6c1e7e30 | ||
|
|
de3afd4199 | ||
|
|
06cd491abc | ||
|
|
7576c244c4 | ||
|
|
a3fae7cd16 | ||
|
|
aa4a22ad47 | ||
|
|
74f60f246f | ||
|
|
9f479aab98 | ||
|
|
bb01b68fa0 | ||
|
|
94b8c42b2c | ||
|
|
8060fdcac5 | ||
|
|
f25e273e32 | ||
|
|
68065128b8 | ||
|
|
8d31e73ffd | ||
|
|
87ccef72db | ||
|
|
ac66227798 | ||
|
|
21b6b07dd8 | ||
|
|
8c4edf7b40 | ||
|
|
8ca7d1ef17 | ||
|
|
3e7e922d16 | ||
|
|
89e276de64 | ||
|
|
a458bee9f2 | ||
|
|
e0eda52c54 | ||
|
|
1d9a888be9 | ||
|
|
32d0413be1 | ||
|
|
01b7aa79fa | ||
|
|
ca836568ac | ||
|
|
3828b8ce87 | ||
|
|
b7e4e26778 | ||
|
|
1f592ae933 | ||
|
|
7e09a548a6 | ||
|
|
ac5ce31278 | ||
|
|
47c72e6f35 | ||
|
|
709c83716c | ||
|
|
f401b996eb | ||
|
|
f18bc2d9dd | ||
|
|
e05641291e | ||
|
|
f6e66f4a19 | ||
|
|
1f99bb4e4e | ||
|
|
53e3536975 | ||
|
|
c88ea31f62 | ||
|
|
da9d3e8f6c | ||
|
|
046c3d5ad4 | ||
|
|
475185fb72 | ||
|
|
ec0edb1ac4 | ||
|
|
03ca0ca2c6 | ||
|
|
93c1030b61 | ||
|
|
0f19ad9e87 | ||
|
|
498599ed55 | ||
|
|
becb987672 | ||
|
|
bfbb9fffa2 | ||
|
|
068b4401fe | ||
|
|
4949464bc7 | ||
|
|
c5244c49a8 | ||
|
|
2ae80c4380 | ||
|
|
2f75a08ce0 | ||
|
|
a730c4211a | ||
|
|
578eb8f39a | ||
|
|
cc7a4b0a6e | ||
|
|
33cff61992 | ||
|
|
dbe4849ffd | ||
|
|
b98c0b4bf4 | ||
|
|
2dc5d4adac | ||
|
|
3d2bbf96d2 | ||
|
|
2807fe08fc | ||
|
|
2b64916d3e | ||
|
|
7386cf6a06 | ||
|
|
40089b6753 | ||
|
|
d24db396cc | ||
|
|
41eeb4deef | ||
|
|
02c2d81cde | ||
|
|
4d5dc55766 | ||
|
|
ebda0689ab | ||
|
|
64f3d6fb6e | ||
|
|
2c8818351f | ||
|
|
a5c94acd4f | ||
|
|
0716092b39 | ||
|
|
994a2dec78 | ||
|
|
8d97add89c | ||
|
|
58c02d6429 | ||
|
|
58f2dd6740 | ||
|
|
f72f696109 | ||
|
|
35261c805e | ||
|
|
6f882eb354 | ||
|
|
7a6c559ab4 | ||
|
|
fc8241fb5e | ||
|
|
c93f7ffa44 | ||
|
|
3c42814b58 | ||
|
|
03a0936556 | ||
|
|
93ca87b7cb | ||
|
|
6147b61b0a | ||
|
|
9ec0d1caaa | ||
|
|
efec31de8d | ||
|
|
3a1ef8837e | ||
|
|
7875470e74 | ||
|
|
c5189c343b | ||
|
|
e2d991d8d0 | ||
|
|
de4fcc0e2c | ||
|
|
fa88dff7ce | ||
|
|
72dca31cfa | ||
|
|
884ffbafee | ||
|
|
39eaf426f8 | ||
|
|
b5e5f6f55a | ||
|
|
867269cdcf | ||
|
|
6f764e1014 | ||
|
|
ba0e3ce5fa | ||
|
|
b6471ecd75 | ||
|
|
6c390851e7 | ||
|
|
312e13416b | ||
|
|
ac694f0c8f | ||
|
|
4820b594a7 | ||
|
|
7b80605538 | ||
|
|
d3f7c09c03 | ||
|
|
11e393dbe7 | ||
|
|
f6a966b4fc | ||
|
|
6cfbfb0849 | ||
|
|
2ae3f812f8 | ||
|
|
9667588af0 | ||
|
|
f0d560060f | ||
|
|
91285bb51d | ||
|
|
b9256e0b34 | ||
|
|
f419d5643a | ||
|
|
bec9f79659 | ||
|
|
99739fa5a9 | ||
|
|
f909265721 | ||
|
|
ffec365ddf | ||
|
|
206715bcd8 | ||
|
|
4c0cd753e9 | ||
|
|
1567ea2f44 | ||
|
|
84ed7ee393 | ||
|
|
2cb1e25ffc | ||
|
|
477ea8fbcf | ||
|
|
d808a2f03d | ||
|
|
aefca613f7 | ||
|
|
2ad92e606a | ||
|
|
9cf68c0cf8 | ||
|
|
d422e7769c | ||
|
|
73a0dea16b | ||
|
|
56ec89625a | ||
|
|
7f672b37e1 | ||
|
|
2b4112e462 | ||
|
|
3c2774ec3d | ||
|
|
1f2c5ee06e | ||
|
|
8050f9202f | ||
|
|
113a4546fd | ||
|
|
7b1497ee63 | ||
|
|
01996bccf8 | ||
|
|
4d5144e50d | ||
|
|
1db012485f | ||
|
|
aca20cd9f4 | ||
|
|
abeb7f65f8 | ||
|
|
da8bab2b33 | ||
|
|
fa61074513 | ||
|
|
3e45d4bec4 | ||
|
|
fbd8f868a1 | ||
|
|
c08f6d0ff7 | ||
|
|
957c9a2eb2 | ||
|
|
300b383cdf | ||
|
|
308a48c0c2 | ||
|
|
1a130125de | ||
|
|
354b7bdff0 | ||
|
|
9c0459685e | ||
|
|
289402a741 | ||
|
|
ab1b9697e6 | ||
|
|
6e90f7bdd0 | ||
|
|
578c8aea0e | ||
|
|
1451b389a4 | ||
|
|
31f9f528db | ||
|
|
392da4e038 | ||
|
|
ad5e8f68ec | ||
|
|
fba5219dab | ||
|
|
6622240df7 | ||
|
|
9a712df6ee | ||
|
|
860c207c94 | ||
|
|
11d79615c8 | ||
|
|
8372721a62 | ||
|
|
26313060a3 | ||
|
|
ee95d23a22 | ||
|
|
fa1e88d94f | ||
|
|
087fa1347f | ||
|
|
de97ba94c3 | ||
|
|
b3076909d1 | ||
|
|
cf241c8901 | ||
|
|
ebcce363e2 | ||
|
|
88dcb17be3 | ||
|
|
d9260f9d04 | ||
|
|
39e4066417 | ||
|
|
e098244afc | ||
|
|
4236c36599 | ||
|
|
7929a50327 | ||
|
|
7dbf19d09a | ||
|
|
37578929d4 | ||
|
|
93ccc6e05f | ||
|
|
f87952a411 | ||
|
|
f5a76f67c1 | ||
|
|
1112dab4f3 | ||
|
|
a77c2232be | ||
|
|
5469efa336 | ||
|
|
24291e23eb | ||
|
|
ef93869009 | ||
|
|
7db6666b97 | ||
|
|
08f2dcc532 | ||
|
|
3d0defa663 | ||
|
|
6f00c7f061 | ||
|
|
d685b94102 | ||
|
|
48382676d2 | ||
|
|
b9518e06f5 | ||
|
|
4e83e92156 | ||
|
|
22e05a2942 | ||
|
|
5b3192344d | ||
|
|
aab3ed2347 | ||
|
|
106dd13306 | ||
|
|
e2ecf137a2 | ||
|
|
185332b91f | ||
|
|
28add5a534 | ||
|
|
d6a5441391 | ||
|
|
1ceab96aba | ||
|
|
e7843b4127 | ||
|
|
cc51c7582f | ||
|
|
c1eb4b2c9f | ||
|
|
3254f18343 | ||
|
|
9f20d3ebd1 | ||
|
|
0f5d7ed242 | ||
|
|
857a42565e | ||
|
|
ffc2c0a2ef | ||
|
|
5536ed85e5 | ||
|
|
54bb41d126 | ||
|
|
7df440e36f | ||
|
|
7c32361a7d |
No files matched your search
@@ -9,12 +9,12 @@ assignees: ''
|
||||
|
||||
Sponsors SeaweedFS via Patreon https://www.patreon.com/seaweedfs
|
||||
Report issues here. Ask questions here https://stackoverflow.com/questions/tagged/seaweedfs
|
||||
Please ask questions in https://github.com/chrislusf/seaweedfs/discussions
|
||||
Please ask questions in https://github.com/seaweedfs/seaweedfs/discussions
|
||||
|
||||
example of a good issue report:
|
||||
https://github.com/chrislusf/seaweedfs/issues/1005
|
||||
https://github.com/seaweedfs/seaweedfs/issues/1005
|
||||
example of a bad issue report:
|
||||
https://github.com/chrislusf/seaweedfs/issues/1008
|
||||
https://github.com/seaweedfs/seaweedfs/issues/1008
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
@@ -5,6 +5,11 @@
|
||||
# How are we solving the problem?
|
||||
|
||||
|
||||
|
||||
# How is the PR tested?
|
||||
|
||||
|
||||
|
||||
# Checks
|
||||
- [ ] I have added unit tests if possible.
|
||||
- [ ] I will add related wiki document changes and link to this PR after merging.
|
||||
@@ -38,13 +38,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -53,14 +53,14 @@ jobs:
|
||||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed-large-disk
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -68,7 +68,7 @@ jobs:
|
||||
release_tag: dev
|
||||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed-normal-disk
|
||||
@@ -87,13 +87,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -102,14 +102,14 @@ jobs:
|
||||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed-large-disk
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -117,7 +117,7 @@ jobs:
|
||||
release_tag: dev
|
||||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed-normal-disk
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -38,13 +38,13 @@ jobs:
|
||||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
# build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -38,13 +38,13 @@ jobs:
|
||||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
# build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -38,13 +38,13 @@ jobs:
|
||||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
# build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -38,13 +38,13 @@ jobs:
|
||||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
# build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
|
||||
@@ -28,9 +28,9 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -39,13 +39,13 @@ jobs:
|
||||
build_flags: -tags elastic,ydb,gocdk,tikv
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
# build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -53,7 +53,7 @@ jobs:
|
||||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
build_flags: -tags 5BytesOffset,elastic,ydb,gocdk,tikv
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
|
||||
@@ -3,6 +3,10 @@ name: "Code Scanning - Action"
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/codeql
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
CodeQL-Build:
|
||||
# CodeQL runs on ubuntu-latest, windows-latest, and macos-latest
|
||||
@@ -14,7 +18,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
@@ -56,7 +56,7 @@ jobs:
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2
|
||||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2
|
||||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2
|
||||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2
|
||||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2
|
||||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2
|
||||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2
|
||||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -9,6 +9,6 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@dcd71f646680f2efd8db4afa5ad64fdcba30e748
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@1c59cdf2a9c7f29c90e8da32237eb04b81bad9f0
|
||||
uses: actions/dependency-review-action@375c5370086bfff256c37f8beec0f437e2e72ae1
|
||||
@@ -0,0 +1,89 @@
|
||||
name: "End to End"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/e2e
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: docker
|
||||
|
||||
jobs:
|
||||
e2e:
|
||||
name: FUSE Mount
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse
|
||||
|
||||
- name: Start SeaweedFS
|
||||
timeout-minutes: 5
|
||||
run: make build_e2e && docker compose -f ./compose/e2e-mount.yml up --wait
|
||||
|
||||
- name: Run FIO
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
echo "Starting FIO at: $(date)"
|
||||
# Concurrent r/w
|
||||
echo 'Run randrw with size=16M bs=4k'
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
|
||||
|
||||
echo 'Run randrw with size=16M bs=128k'
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
|
||||
|
||||
echo 'Run randrw with size=16M bs=1m'
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
|
||||
|
||||
# Verified write
|
||||
echo 'Run randwrite with size=16M bs=4k'
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
|
||||
|
||||
echo 'Run randwrite with size=16M bs=128k'
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
|
||||
|
||||
echo 'Run randwrite with size=16M bs=1m'
|
||||
docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 40 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=1m --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
|
||||
|
||||
- name: Save logs
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f ./compose/e2e-mount.yml logs > output.log
|
||||
echo 'Showing last 500 log lines of mount service:'
|
||||
docker compose -f ./compose/e2e-mount.yml logs --tail 500 mount
|
||||
|
||||
- name: Check for data races
|
||||
if: always()
|
||||
continue-on-error: true # TODO: remove this comment to enable build failure on data races (after all are fixed)
|
||||
run: grep -A50 'DATA RACE' output.log && exit 1 || exit 0
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: output-logs
|
||||
path: docker/output.log
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: docker compose -f ./compose/e2e-mount.yml down --volumes --remove-orphans --rmi all
|
||||
@@ -21,13 +21,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@b22fbbc2921299758641fab08929b4ac52b32923 # v2
|
||||
uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
|
||||
@@ -10,5 +10,5 @@ install:
|
||||
full_install:
|
||||
cd weed; go install -tags "elastic gocdk sqlite ydb tikv"
|
||||
|
||||
test:
|
||||
tests:
|
||||
cd weed; go test -tags "elastic gocdk sqlite ydb tikv" -v ./...
|
||||
@@ -3,19 +3,19 @@
|
||||
|
||||
[](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
|
||||
[](https://twitter.com/intent/follow?screen_name=seaweedfs)
|
||||
[](https://github.com/chrislusf/seaweedfs/actions/workflows/go.yml)
|
||||
[](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
|
||||
[](https://github.com/chrislusf/seaweedfs/wiki)
|
||||
[](https://github.com/seaweedfs/seaweedfs/actions/workflows/go.yml)
|
||||
[](https://godoc.org/github.com/seaweedfs/seaweedfs/weed)
|
||||
[](https://github.com/seaweedfs/seaweedfs/wiki)
|
||||
[](https://hub.docker.com/r/chrislusf/seaweedfs/)
|
||||
[](https://search.maven.org/search?q=g:com.github.chrislusf)
|
||||
|
||||
|
||||

|
||||

|
||||
|
||||
<h2 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a></h2>
|
||||
|
||||
SeaweedFS is an independent Apache-licensed open source project with its ongoing development made
|
||||
possible entirely thanks to the support of these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md).
|
||||
possible entirely thanks to the support of these awesome [backers](https://github.com/seaweedfs/seaweedfs/blob/master/backers.md).
|
||||
If you'd like to grow SeaweedFS even stronger, please consider joining our
|
||||
<a href="https://www.patreon.com/seaweedfs">sponsors on Patreon</a>.
|
||||
|
||||
@@ -32,18 +32,19 @@ Your support will be really appreciated by me and other supporters!
|
||||
-->
|
||||
|
||||
### Gold Sponsors
|
||||
- [](https://www.nodion.com)
|
||||
[](https://www.nodion.com)
|
||||
[](https://www.piknik.com)
|
||||
|
||||
---
|
||||
|
||||
- [Download Binaries for different platforms](https://github.com/chrislusf/seaweedfs/releases/latest)
|
||||
- [Download Binaries for different platforms](https://github.com/seaweedfs/seaweedfs/releases/latest)
|
||||
- [SeaweedFS on Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
|
||||
- [SeaweedFS on Twitter](https://twitter.com/SeaweedFS)
|
||||
- [SeaweedFS on Telegram](https://t.me/Seaweedfs)
|
||||
- [SeaweedFS on Reddit](https://www.reddit.com/r/SeaweedFS/)
|
||||
- [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
|
||||
- [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
|
||||
- [SeaweedFS White Paper](https://github.com/chrislusf/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
|
||||
- [Wiki Documentation](https://github.com/seaweedfs/seaweedfs/wiki)
|
||||
- [SeaweedFS White Paper](https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
|
||||
- [SeaweedFS Introduction Slides 2021.5](https://docs.google.com/presentation/d/1DcxKWlINc-HNCjhYeERkpGXXm6nTCES8mi2W5G0Z4Ts/edit?usp=sharing)
|
||||
- [SeaweedFS Introduction Slides 2019.3](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
|
||||
|
||||
@@ -58,8 +59,8 @@ Table of Contents
|
||||
* [Features](#features)
|
||||
* [Additional Features](#additional-features)
|
||||
* [Filer Features](#filer-features)
|
||||
* [Example: Using Seaweed Object Store](#example-Using-Seaweed-Object-Store)
|
||||
* [Architecture](#Object-Store-Architecture)
|
||||
* [Example: Using Seaweed Object Store](#example-using-seaweed-object-store)
|
||||
* [Architecture](#object-store-architecture)
|
||||
* [Compared to Other File Systems](#compared-to-other-file-systems)
|
||||
* [Compared to HDFS](#compared-to-hdfs)
|
||||
* [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph)
|
||||
@@ -68,16 +69,17 @@ Table of Contents
|
||||
* [Dev Plan](#dev-plan)
|
||||
* [Installation Guide](#installation-guide)
|
||||
* [Disk Related Topics](#disk-related-topics)
|
||||
* [Benchmark](#Benchmark)
|
||||
* [Benchmark](#benchmark)
|
||||
* [License](#license)
|
||||
|
||||
# Quick Start #
|
||||
|
||||
## Quick Start for S3 API on Docker ##
|
||||
|
||||
`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
|
||||
|
||||
## Quick Start with Single Binary ##
|
||||
* Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
|
||||
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
|
||||
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
|
||||
|
||||
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
|
||||
@@ -85,7 +87,7 @@ Also, to increase capacity, just add more volume servers by running `weed volume
|
||||
## Quick Start SeaweedFS S3 on AWS ##
|
||||
* Setup fast production-ready [SeaweedFS S3 on AWS with cloudformation](https://aws.amazon.com/marketplace/pp/prodview-nzelz5gprlrjc)
|
||||
|
||||
## Introduction ##
|
||||
# Introduction #
|
||||
|
||||
SeaweedFS is a simple and highly scalable distributed file system. There are two objectives:
|
||||
|
||||
@@ -122,6 +124,7 @@ Faster and Cheaper than direct cloud storage!
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
# Features #
|
||||
## Additional Features ##
|
||||
* Can choose no replication or different replication levels, rack and data center aware.
|
||||
* Automatic master servers failover - no single point of failure (SPOF).
|
||||
@@ -159,25 +162,25 @@ Faster and Cheaper than direct cloud storage!
|
||||
* [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
|
||||
* [SeaweedFS Operator](https://github.com/seaweedfs/seaweedfs-operator)
|
||||
|
||||
[Filer]: https://github.com/chrislusf/seaweedfs/wiki/Directories-and-Files
|
||||
[SuperLargeFiles]: https://github.com/chrislusf/seaweedfs/wiki/Data-Structure-for-Large-Files
|
||||
[Mount]: https://github.com/chrislusf/seaweedfs/wiki/FUSE-Mount
|
||||
[AmazonS3API]: https://github.com/chrislusf/seaweedfs/wiki/Amazon-S3-API
|
||||
[BackupToCloud]: https://github.com/chrislusf/seaweedfs/wiki/Async-Replication-to-Cloud
|
||||
[Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System
|
||||
[WebDAV]: https://github.com/chrislusf/seaweedfs/wiki/WebDAV
|
||||
[ErasureCoding]: https://github.com/chrislusf/seaweedfs/wiki/Erasure-coding-for-warm-storage
|
||||
[TieredStorage]: https://github.com/chrislusf/seaweedfs/wiki/Tiered-Storage
|
||||
[CloudTier]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Tier
|
||||
[FilerDataEncryption]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Data-Encryption
|
||||
[FilerTTL]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Stores
|
||||
[VolumeServerTTL]: https://github.com/chrislusf/seaweedfs/wiki/Store-file-with-a-Time-To-Live
|
||||
[Filer]: https://github.com/seaweedfs/seaweedfs/wiki/Directories-and-Files
|
||||
[SuperLargeFiles]: https://github.com/seaweedfs/seaweedfs/wiki/Data-Structure-for-Large-Files
|
||||
[Mount]: https://github.com/seaweedfs/seaweedfs/wiki/FUSE-Mount
|
||||
[AmazonS3API]: https://github.com/seaweedfs/seaweedfs/wiki/Amazon-S3-API
|
||||
[BackupToCloud]: https://github.com/seaweedfs/seaweedfs/wiki/Async-Replication-to-Cloud
|
||||
[Hadoop]: https://github.com/seaweedfs/seaweedfs/wiki/Hadoop-Compatible-File-System
|
||||
[WebDAV]: https://github.com/seaweedfs/seaweedfs/wiki/WebDAV
|
||||
[ErasureCoding]: https://github.com/seaweedfs/seaweedfs/wiki/Erasure-coding-for-warm-storage
|
||||
[TieredStorage]: https://github.com/seaweedfs/seaweedfs/wiki/Tiered-Storage
|
||||
[CloudTier]: https://github.com/seaweedfs/seaweedfs/wiki/Cloud-Tier
|
||||
[FilerDataEncryption]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Data-Encryption
|
||||
[FilerTTL]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Stores
|
||||
[VolumeServerTTL]: https://github.com/seaweedfs/seaweedfs/wiki/Store-file-with-a-Time-To-Live
|
||||
[SeaweedFsCsiDriver]: https://github.com/seaweedfs/seaweedfs-csi-driver
|
||||
[ActiveActiveAsyncReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Active-Active-cross-cluster-continuous-synchronization
|
||||
[FilerStoreReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Store-Replication
|
||||
[KeyLargeValueStore]: https://github.com/chrislusf/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
|
||||
[CloudDrive]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Drive-Architecture
|
||||
[GatewayToRemoteObjectStore]: https://github.com/chrislusf/seaweedfs/wiki/Gateway-to-Remote-Object-Storage
|
||||
[ActiveActiveAsyncReplication]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Active-Active-cross-cluster-continuous-synchronization
|
||||
[FilerStoreReplication]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication
|
||||
[KeyLargeValueStore]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
|
||||
[CloudDrive]: https://github.com/seaweedfs/seaweedfs/wiki/Cloud-Drive-Architecture
|
||||
[GatewayToRemoteObjectStore]: https://github.com/seaweedfs/seaweedfs/wiki/Gateway-to-Remote-Object-Storage
|
||||
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
@@ -300,7 +303,7 @@ The replication parameter options are:
|
||||
|
||||
More details about replication can be found [on the wiki][Replication].
|
||||
|
||||
[Replication]: https://github.com/chrislusf/seaweedfs/wiki/Replication
|
||||
[Replication]: https://github.com/seaweedfs/seaweedfs/wiki/Replication
|
||||
|
||||
You can also set the default replication strategy when starting the master server.
|
||||
|
||||
@@ -325,10 +328,10 @@ When requesting a file key, an optional "dataCenter" parameter can limit the ass
|
||||
* [Chunking large files][feat-3]
|
||||
* [Collection as a Simple Name Space][feat-4]
|
||||
|
||||
[feat-1]: https://github.com/chrislusf/seaweedfs/wiki/Failover-Master-Server
|
||||
[feat-2]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#insert-with-your-own-keys
|
||||
[feat-3]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#upload-large-files
|
||||
[feat-4]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#collection-as-a-simple-name-space
|
||||
[feat-1]: https://github.com/seaweedfs/seaweedfs/wiki/Failover-Master-Server
|
||||
[feat-2]: https://github.com/seaweedfs/seaweedfs/wiki/Optimization#insert-with-your-own-keys
|
||||
[feat-3]: https://github.com/seaweedfs/seaweedfs/wiki/Optimization#upload-large-files
|
||||
[feat-4]: https://github.com/seaweedfs/seaweedfs/wiki/Optimization#collection-as-a-simple-name-space
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
@@ -442,7 +445,7 @@ MooseFS Master Server keeps all meta data in memory. Same issue as HDFS namenode
|
||||
|
||||
### Compared to Ceph ###
|
||||
|
||||
Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more complicated, with the need to support layers on top of it. [Here is a more detailed comparison](https://github.com/chrislusf/seaweedfs/issues/120)
|
||||
Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more complicated, with the need to support layers on top of it. [Here is a more detailed comparison](https://github.com/seaweedfs/seaweedfs/issues/120)
|
||||
|
||||
SeaweedFS has a centralized master group to look up free volumes, while Ceph uses hashing and metadata servers to locate its objects. Having a centralized master makes it easy to code and manage.
|
||||
|
||||
@@ -502,7 +505,7 @@ make sure to define your $GOPATH
|
||||
|
||||
Step 2: checkout this repo:
|
||||
```bash
|
||||
git clone https://github.com/chrislusf/seaweedfs.git
|
||||
git clone https://github.com/seaweedfs/seaweedfs.git
|
||||
```
|
||||
Step 3: download, compile, and install the project by executing the following command
|
||||
|
||||
@@ -601,4 +604,3 @@ The text of this page is available for modification and reuse under the terms of
|
||||
## Stargazers over time
|
||||
|
||||
[](https://starchart.cc/chrislusf/seaweedfs)
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
FROM ubuntu:22.04
|
||||
|
||||
LABEL author="Chris Lu"
|
||||
|
||||
RUN apt-get update && apt-get install -y curl fio fuse
|
||||
RUN mkdir -p /etc/seaweedfs /data/filerldb2
|
||||
|
||||
COPY ./weed /usr/bin/
|
||||
COPY ./filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY ./entrypoint.sh /entrypoint.sh
|
||||
|
||||
# volume server grpc port
|
||||
EXPOSE 18080
|
||||
# volume server http port
|
||||
EXPOSE 8080
|
||||
# filer server grpc port
|
||||
EXPOSE 18888
|
||||
# filer server http port
|
||||
EXPOSE 8888
|
||||
# master server shared grpc port
|
||||
EXPOSE 19333
|
||||
# master server shared http port
|
||||
EXPOSE 9333
|
||||
|
||||
VOLUME /data
|
||||
WORKDIR /data
|
||||
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
@@ -1,20 +1,20 @@
|
||||
FROM gcc:11 as builder
|
||||
RUN mkdir -p /go/src/github.com/chrislusf/
|
||||
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
|
||||
RUN mkdir -p /go/src/github.com/seaweedfs/
|
||||
RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
|
||||
ARG BRANCH=${BRANCH:-master}
|
||||
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
|
||||
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y golang-src \
|
||||
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& CGO_ENABLED=0 go install -ldflags "-extldflags -static ${LDFLAGS}" -compiler=gccgo -tags gccgo,noasm
|
||||
|
||||
FROM alpine AS final
|
||||
LABEL author="Chris Lu"
|
||||
COPY --from=builder /go/bin/weed /usr/bin/
|
||||
RUN mkdir -p /etc/seaweedfs
|
||||
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
RUN apk add fuse # for weed mount
|
||||
|
||||
# volume server gprc port
|
||||
@@ -41,4 +41,4 @@ WORKDIR /data
|
||||
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
@@ -1,20 +1,20 @@
|
||||
FROM golang:1.18-alpine as builder
|
||||
FROM golang:1.19-alpine as builder
|
||||
RUN apk add git g++ fuse
|
||||
RUN mkdir -p /go/src/github.com/chrislusf/
|
||||
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
|
||||
RUN mkdir -p /go/src/github.com/seaweedfs/
|
||||
RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
|
||||
ARG BRANCH=${BRANCH:-master}
|
||||
ARG TAGS
|
||||
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
|
||||
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
|
||||
|
||||
FROM alpine AS final
|
||||
LABEL author="Chris Lu"
|
||||
COPY --from=builder /go/bin/weed /usr/bin/
|
||||
RUN mkdir -p /etc/seaweedfs
|
||||
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
RUN apk add fuse # for weed mount
|
||||
|
||||
# volume server gprc port
|
||||
|
||||
@@ -5,6 +5,7 @@ RUN mkdir -p /etc/seaweedfs
|
||||
COPY ./filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY ./entrypoint.sh /entrypoint.sh
|
||||
RUN apk add fuse # for weed mount
|
||||
RUN apk add curl # for health checks
|
||||
|
||||
# volume server grpc port
|
||||
EXPOSE 18080
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
FROM golang:1.19-buster as builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
ENV ROCKSDB_VERSION v7.5.3
|
||||
|
||||
# build RocksDB
|
||||
RUN cd /tmp && \
|
||||
git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \
|
||||
cd rocksdb && \
|
||||
PORTABLE=1 make static_lib && \
|
||||
make install-static
|
||||
|
||||
ENV CGO_CFLAGS "-I/tmp/rocksdb/include"
|
||||
ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
|
||||
@@ -1,9 +1,9 @@
|
||||
FROM golang:1.18-buster as builder
|
||||
FROM golang:1.19-buster as builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
ENV ROCKSDB_VERSION v7.2.2
|
||||
ENV ROCKSDB_VERSION v7.5.3
|
||||
|
||||
# build RocksDB
|
||||
RUN cd /tmp && \
|
||||
@@ -16,12 +16,12 @@ ENV CGO_CFLAGS "-I/tmp/rocksdb/include"
|
||||
ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
|
||||
|
||||
# build SeaweedFS
|
||||
RUN mkdir -p /go/src/github.com/chrislusf/
|
||||
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
|
||||
RUN mkdir -p /go/src/github.com/seaweedfs/
|
||||
RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
|
||||
ARG BRANCH=${BRANCH:-master}
|
||||
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
|
||||
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}"
|
||||
|
||||
|
||||
@@ -29,8 +29,8 @@ FROM alpine AS final
|
||||
LABEL author="Chris Lu"
|
||||
COPY --from=builder /go/bin/weed /usr/bin/
|
||||
RUN mkdir -p /etc/seaweedfs
|
||||
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
RUN apk add fuse snappy gflags
|
||||
|
||||
# volume server gprc port
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
FROM chrislusf/rocksdb_dev_env as builder
|
||||
|
||||
# build SeaweedFS
|
||||
RUN mkdir -p /go/src/github.com/seaweedfs/
|
||||
ADD . /go/src/github.com/seaweedfs/seaweedfs
|
||||
RUN ls -al /go/src/github.com/seaweedfs/ && \
|
||||
cd /go/src/github.com/seaweedfs/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}"
|
||||
|
||||
|
||||
FROM alpine AS final
|
||||
LABEL author="Chris Lu"
|
||||
COPY --from=builder /go/bin/weed /usr/bin/
|
||||
RUN mkdir -p /etc/seaweedfs
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
RUN apk add fuse snappy gflags tmux
|
||||
|
||||
# volume server gprc port
|
||||
EXPOSE 18080
|
||||
# volume server http port
|
||||
EXPOSE 8080
|
||||
# filer server gprc port
|
||||
EXPOSE 18888
|
||||
# filer server http port
|
||||
EXPOSE 8888
|
||||
# master server shared gprc port
|
||||
EXPOSE 19333
|
||||
# master server shared http port
|
||||
EXPOSE 9333
|
||||
# s3 server http port
|
||||
EXPOSE 8333
|
||||
# webdav server http port
|
||||
EXPOSE 7333
|
||||
|
||||
RUN mkdir -p /data/filer_rocksdb
|
||||
|
||||
VOLUME /data
|
||||
|
||||
WORKDIR /data
|
||||
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
@@ -28,4 +28,4 @@ ENV \
|
||||
S3TEST_CONF="/s3test.conf"
|
||||
|
||||
ENTRYPOINT ["/bin/bash", "-c"]
|
||||
CMD ["sleep 30 && exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"]
|
||||
CMD ["sleep 30 && exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"]
|
||||
+19
-3
@@ -4,14 +4,21 @@ all: gen
|
||||
|
||||
gen: dev
|
||||
|
||||
cgo ?= 0
|
||||
binary:
|
||||
export SWCOMMIT=$(shell git rev-parse --short HEAD)
|
||||
export SWLDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
|
||||
cd ../weed; CGO_ENABLED=0 GOOS=linux go build -tags "$(tags)" -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/
|
||||
export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
|
||||
cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-extldflags -static $(SWLDFLAGS)" && mv weed ../docker/
|
||||
|
||||
binary_race: options = -race
|
||||
binary_race: cgo = 1
|
||||
binary_race: binary
|
||||
|
||||
build: binary
|
||||
docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
|
||||
rm ./weed
|
||||
|
||||
build_e2e: binary_race
|
||||
docker build --no-cache -t chrislusf/seaweedfs:e2e -f Dockerfile.e2e .
|
||||
|
||||
go_build: # make go_build tags=elastic,ydb,gocdk,hdfs,5BytesOffset
|
||||
docker build --build-arg TAGS=$(tags) --no-cache -t chrislusf/seaweedfs:go_build -f Dockerfile.go_build .
|
||||
@@ -19,6 +26,12 @@ go_build: # make go_build tags=elastic,ydb,gocdk,hdfs,5BytesOffset
|
||||
go_build_large_disk:
|
||||
docker build --build-arg TAGS=large_disk --no-cache -t chrislusf/seaweedfs:large_disk -f Dockerfile.go_build .
|
||||
|
||||
build_rocksdb_dev_env:
|
||||
docker build --no-cache -t chrislusf/rocksdb_dev_env -f Dockerfile.rocksdb_dev_env .
|
||||
|
||||
build_rocksdb_local:
|
||||
cd .. ; docker build --no-cache -t chrislusf/seaweedfs:rocksdb_local -f docker/Dockerfile.rocksdb_large_local .
|
||||
|
||||
build_rocksdb:
|
||||
docker build --no-cache -t chrislusf/seaweedfs:rocksdb -f Dockerfile.rocksdb_large .
|
||||
|
||||
@@ -28,6 +41,9 @@ s3tests_build:
|
||||
dev: build
|
||||
docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
|
||||
|
||||
dev_race: binary_race
|
||||
docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
|
||||
|
||||
dev_tls: build certstrap
|
||||
ENV_FILE="tls.env" docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
|
||||
|
||||
|
||||
+5
-5
@@ -5,7 +5,7 @@
|
||||
|
||||
```bash
|
||||
|
||||
wget https://raw.githubusercontent.com/chrislusf/seaweedfs/master/docker/seaweedfs-compose.yml
|
||||
wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-compose.yml
|
||||
|
||||
docker-compose -f seaweedfs-compose.yml -p seaweedfs up
|
||||
|
||||
@@ -15,7 +15,7 @@ docker-compose -f seaweedfs-compose.yml -p seaweedfs up
|
||||
|
||||
```bash
|
||||
|
||||
wget https://raw.githubusercontent.com/chrislusf/seaweedfs/master/docker/seaweedfs-dev-compose.yml
|
||||
wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-dev-compose.yml
|
||||
|
||||
docker-compose -f seaweedfs-dev-compose.yml -p seaweedfs up
|
||||
|
||||
@@ -24,7 +24,7 @@ docker-compose -f seaweedfs-dev-compose.yml -p seaweedfs up
|
||||
## Local Development
|
||||
|
||||
```bash
|
||||
cd $GOPATH/src/github.com/chrislusf/seaweedfs/docker
|
||||
cd $GOPATH/src/github.com/seaweedfs/seaweedfs/docker
|
||||
make
|
||||
```
|
||||
|
||||
@@ -44,8 +44,8 @@ docker buildx build --pull --push --platform linux/386,linux/amd64,linux/arm64,l
|
||||
docker buildx stop $BUILDER
|
||||
```
|
||||
|
||||
## Minio debuging
|
||||
## Minio debugging
|
||||
```
|
||||
mc config host add local http://127.0.0.1:9000 some_access_key1 some_secret_key1
|
||||
mc admin trace --all --verbose local
|
||||
```
|
||||
```
|
||||
@@ -0,0 +1,53 @@
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
master:
|
||||
image: chrislusf/seaweedfs:e2e
|
||||
command: "-v=4 master -ip=master -ip.bind=0.0.0.0 -raftBootstrap"
|
||||
healthcheck:
|
||||
test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ]
|
||||
interval: 1s
|
||||
timeout: 60s
|
||||
|
||||
volume:
|
||||
image: chrislusf/seaweedfs:e2e
|
||||
command: "-v=4 volume -mserver=master:9333 -ip=volume -ip.bind=0.0.0.0 -preStopSeconds=1"
|
||||
healthcheck:
|
||||
test: [ "CMD", "curl", "--fail", "-I", "http://localhost:8080/healthz" ]
|
||||
interval: 1s
|
||||
timeout: 30s
|
||||
depends_on:
|
||||
master:
|
||||
condition: service_healthy
|
||||
|
||||
filer:
|
||||
image: chrislusf/seaweedfs:e2e
|
||||
command: "-v=4 filer -master=master:9333 -ip=filer -ip.bind=0.0.0.0"
|
||||
healthcheck:
|
||||
test: [ "CMD", "curl", "--fail", "-I", "http://localhost:8888" ]
|
||||
interval: 1s
|
||||
timeout: 30s
|
||||
depends_on:
|
||||
volume:
|
||||
condition: service_healthy
|
||||
|
||||
mount:
|
||||
image: chrislusf/seaweedfs:e2e
|
||||
command: "-v=4 mount -filer=filer:8888 -filer.path=/ -dirAutoCreate -dir=/mnt/seaweedfs"
|
||||
cap_add:
|
||||
- SYS_ADMIN
|
||||
devices:
|
||||
- /dev/fuse
|
||||
security_opt:
|
||||
- apparmor:unconfined
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4096m
|
||||
healthcheck:
|
||||
test: [ "CMD", "mountpoint", "-q", "--", "/mnt/seaweedfs" ]
|
||||
interval: 1s
|
||||
timeout: 30s
|
||||
depends_on:
|
||||
filer:
|
||||
condition: service_healthy
|
||||
@@ -6,7 +6,7 @@ services:
|
||||
ports:
|
||||
- 9333:9333
|
||||
- 19333:19333
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data"
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data"
|
||||
volumes:
|
||||
- ./master/0:/data
|
||||
environment:
|
||||
@@ -18,7 +18,7 @@ services:
|
||||
ports:
|
||||
- 9334:9334
|
||||
- 19334:19334
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data"
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data"
|
||||
volumes:
|
||||
- ./master/1:/data
|
||||
environment:
|
||||
@@ -30,7 +30,7 @@ services:
|
||||
ports:
|
||||
- 9335:9335
|
||||
- 19335:19335
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data"
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data"
|
||||
volumes:
|
||||
- ./master/2:/data
|
||||
environment:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[notification.log]
|
||||
# this is only for debugging perpose and does not work with "weed filer.replicate"
|
||||
# this is only for debugging purpose and does not work with "weed filer.replicate"
|
||||
enabled = false
|
||||
|
||||
|
||||
|
||||
@@ -1,29 +1,25 @@
|
||||
module github.com/chrislusf/seaweedfs
|
||||
module github.com/seaweedfs/seaweedfs
|
||||
|
||||
go 1.18
|
||||
go 1.19
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.102.1 // indirect
|
||||
cloud.google.com/go/pubsub v1.23.1
|
||||
cloud.google.com/go/storage v1.23.0
|
||||
cloud.google.com/go v0.104.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.25.1
|
||||
cloud.google.com/go/storage v1.27.0
|
||||
github.com/Azure/azure-pipeline-go v0.2.3
|
||||
github.com/Azure/azure-storage-blob-go v0.15.0
|
||||
github.com/OneOfOne/xxhash v1.2.8
|
||||
github.com/Shopify/sarama v1.34.1
|
||||
github.com/aws/aws-sdk-go v1.44.47
|
||||
github.com/Shopify/sarama v1.37.0
|
||||
github.com/aws/aws-sdk-go v1.44.110
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
|
||||
github.com/bwmarrin/snowflake v0.3.0
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/chrislusf/raft v1.0.9
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
github.com/dustin/go-humanize v1.0.0
|
||||
github.com/eapache/go-resiliency v1.2.0 // indirect
|
||||
github.com/eapache/go-resiliency v1.3.0 // indirect
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
|
||||
github.com/eapache/queue v1.1.0 // indirect
|
||||
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a
|
||||
@@ -31,13 +27,12 @@ require (
|
||||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
|
||||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fclairamb/ftpserverlib v0.18.0
|
||||
github.com/fclairamb/ftpserverlib v0.20.0
|
||||
github.com/fsnotify/fsnotify v1.5.4 // indirect
|
||||
github.com/go-errors/errors v1.1.1 // indirect
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/go-redsync/redsync/v4 v4.5.1
|
||||
github.com/go-redsync/redsync/v4 v4.6.0
|
||||
github.com/go-sql-driver/mysql v1.6.0
|
||||
github.com/go-stack/stack v1.8.1 // indirect
|
||||
github.com/go-zookeeper/zk v1.0.2 // indirect
|
||||
github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible
|
||||
@@ -48,30 +43,30 @@ require (
|
||||
github.com/google/go-cmp v0.5.8 // indirect
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/google/wire v0.5.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.4.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.5.1 // indirect
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.2 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/jcmturner/gofork v1.0.0 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect
|
||||
github.com/jcmturner/gofork v1.7.6 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect
|
||||
github.com/jinzhu/copier v0.3.5
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/klauspost/compress v1.15.6 // indirect
|
||||
github.com/klauspost/reedsolomon v1.10.0
|
||||
github.com/klauspost/compress v1.15.11 // indirect
|
||||
github.com/klauspost/reedsolomon v1.11.0
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/lib/pq v1.10.6
|
||||
github.com/linxGnu/grocksdb v1.7.3
|
||||
github.com/lib/pq v1.10.7
|
||||
github.com/linxGnu/grocksdb v1.7.7
|
||||
github.com/magiconair/properties v1.8.6 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.3 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.9 // indirect
|
||||
github.com/mattn/go-isatty v0.0.16 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
@@ -82,24 +77,23 @@ require (
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/posener/complete v1.2.3
|
||||
github.com/pquerna/cachecontrol v0.1.0
|
||||
github.com/prometheus/client_golang v1.12.2
|
||||
github.com/prometheus/client_golang v1.13.0
|
||||
github.com/prometheus/client_model v0.2.0 // indirect
|
||||
github.com/prometheus/common v0.32.1 // indirect
|
||||
github.com/prometheus/procfs v0.7.3 // indirect
|
||||
github.com/prometheus/common v0.37.0 // indirect
|
||||
github.com/prometheus/procfs v0.8.0
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
|
||||
github.com/seaweedfs/goexif v2.0.0+incompatible
|
||||
github.com/sirupsen/logrus v1.8.1 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/spf13/afero v1.8.2 // indirect
|
||||
github.com/seaweedfs/raft v1.1.0
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/spf13/afero v1.9.2 // indirect
|
||||
github.com/spf13/cast v1.5.0 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/spf13/viper v1.12.0
|
||||
github.com/streadway/amqp v1.0.0
|
||||
github.com/spf13/viper v1.13.0
|
||||
github.com/stretchr/testify v1.8.0
|
||||
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
|
||||
github.com/tidwall/gjson v1.14.1
|
||||
github.com/tidwall/gjson v1.14.3
|
||||
github.com/tidwall/match v1.1.1
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
|
||||
@@ -112,49 +106,52 @@ require (
|
||||
github.com/xdg-go/scram v1.1.1 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.3 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.5.4
|
||||
go.mongodb.org/mongo-driver v1.9.1
|
||||
go.etcd.io/etcd/client/v3 v3.5.5
|
||||
go.mongodb.org/mongo-driver v1.10.2
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
gocloud.dev v0.25.0
|
||||
gocloud.dev/pubsub/natspubsub v0.25.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.25.0
|
||||
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
|
||||
gocloud.dev v0.27.0
|
||||
gocloud.dev/pubsub/natspubsub v0.27.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.27.0
|
||||
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
|
||||
golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd
|
||||
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
|
||||
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e
|
||||
golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 // indirect
|
||||
golang.org/x/sys v0.0.0-20220624220833-87e55d714810
|
||||
golang.org/x/net v0.0.0-20220927171203-f486391704dc
|
||||
golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 // indirect
|
||||
golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023
|
||||
golang.org/x/tools v0.1.12
|
||||
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
|
||||
google.golang.org/api v0.86.0
|
||||
google.golang.org/api v0.98.0
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f // indirect
|
||||
google.golang.org/grpc v1.47.0
|
||||
google.golang.org/protobuf v1.28.0
|
||||
google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006 // indirect
|
||||
google.golang.org/grpc v1.49.0
|
||||
google.golang.org/protobuf v1.28.1
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/cc/v3 v3.36.0 // indirect
|
||||
modernc.org/ccgo/v3 v3.16.6 // indirect
|
||||
modernc.org/libc v1.16.7 // indirect
|
||||
modernc.org/mathutil v1.4.1 // indirect
|
||||
modernc.org/memory v1.1.1 // indirect
|
||||
modernc.org/opt v0.1.1 // indirect
|
||||
modernc.org/sqlite v1.17.3
|
||||
modernc.org/strutil v1.1.2
|
||||
modernc.org/token v1.0.0 // indirect
|
||||
modernc.org/cc/v3 v3.38.1 // indirect
|
||||
modernc.org/ccgo/v3 v3.16.9 // indirect
|
||||
modernc.org/libc v1.19.0 // indirect
|
||||
modernc.org/mathutil v1.5.0 // indirect
|
||||
modernc.org/memory v1.4.0 // indirect
|
||||
modernc.org/opt v0.1.3 // indirect
|
||||
modernc.org/sqlite v1.19.1
|
||||
modernc.org/strutil v1.1.3
|
||||
modernc.org/token v1.0.1 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Jille/raft-grpc-transport v1.2.0
|
||||
github.com/arangodb/go-driver v1.3.2
|
||||
github.com/arangodb/go-driver v1.3.3
|
||||
github.com/fluent/fluent-logger-golang v1.9.0
|
||||
github.com/google/flatbuffers v22.9.29+incompatible
|
||||
github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17
|
||||
github.com/hashicorp/raft v1.3.9
|
||||
github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0
|
||||
github.com/hashicorp/raft v1.3.10
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.2.2
|
||||
github.com/rabbitmq/amqp091-go v1.5.0
|
||||
github.com/tikv/client-go/v2 v2.0.1
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.27.0
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.38.3
|
||||
golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7
|
||||
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
|
||||
)
|
||||
|
||||
@@ -163,30 +160,30 @@ require (
|
||||
cloud.google.com/go/iam v0.3.0 // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/armon/go-metrics v0.3.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.16.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.15.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.11.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.17.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect
|
||||
github.com/aws/smithy-go v1.11.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.16.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.15.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.12.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.17.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.19.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.16.10 // indirect
|
||||
github.com/aws/smithy-go v1.12.0 // indirect
|
||||
github.com/benbjohnson/clock v1.1.0 // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
|
||||
github.com/fatih/color v1.13.0 // indirect
|
||||
github.com/fclairamb/go-log v0.3.0 // indirect
|
||||
github.com/fclairamb/go-log v0.4.1 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
|
||||
github.com/googleapis/go-type-adapters v1.0.0 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
|
||||
github.com/hashicorp/go-hclog v1.2.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-msgpack v1.1.5 // indirect
|
||||
@@ -196,43 +193,44 @@ require (
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/jonboulle/clockwork v0.2.2 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.0.14 // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.7 // indirect
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
|
||||
github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d // indirect
|
||||
github.com/montanaflynn/stats v0.6.6 // indirect
|
||||
github.com/nats-io/nats.go v1.16.0 // indirect
|
||||
github.com/nats-io/nkeys v0.3.0 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.1 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.5 // indirect
|
||||
github.com/philhofer/fwd v1.1.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.14 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.17 // indirect
|
||||
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898 // indirect
|
||||
github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee // indirect
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/subosito/gotenv v1.3.0 // indirect
|
||||
github.com/subosito/gotenv v1.4.1 // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201 // indirect
|
||||
github.com/tinylib/msgp v1.1.6 // indirect
|
||||
github.com/twmb/murmur3 v1.1.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220531094121-36ca6bddb9f7 // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220922065549-66df47a830ba // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.5.4 // indirect
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
|
||||
go.etcd.io/bbolt v1.3.6 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.5.5 // indirect
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.8.0 // indirect
|
||||
go.uber.org/zap v1.21.0 // indirect
|
||||
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
|
||||
gopkg.in/ini.v1 v1.66.4 // indirect
|
||||
go.uber.org/zap v1.22.0 // indirect
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
lukechampine.com/uint128 v1.1.1 // indirect
|
||||
)
|
||||
|
||||
// replace github.com/chrislusf/raft => /Users/chrislu/go/src/github.com/chrislusf/raft
|
||||
// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
|
||||
@@ -1,5 +1,5 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "3.14"
|
||||
version: "3.14"
|
||||
appVersion: "3.31"
|
||||
version: "3.31"
|
||||
@@ -1,6 +1,6 @@
|
||||
## SEAWEEDFS - helm chart (2.x)
|
||||
# SEAWEEDFS - helm chart (2.x)
|
||||
|
||||
### info:
|
||||
## Info:
|
||||
* master/filer/volume are stateful sets with anti-affinity on the hostname,
|
||||
so your deployment will be spread/HA.
|
||||
* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances)
|
||||
@@ -9,12 +9,26 @@ and backup/HA memsql can provide.
|
||||
with ENV.
|
||||
* cert config exists and can be enabled, but not been tested.
|
||||
|
||||
### prerequisites
|
||||
kubernetes node have labels which help to define which node(Host) will run which pod.
|
||||
## Prerequisites
|
||||
### Database
|
||||
A running MySQL-compatible database is expected by default, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
|
||||
This database should be pre-configured and initialized by running:
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS filemeta (
|
||||
dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
|
||||
name VARCHAR(1000) BINARY COMMENT 'directory or file name',
|
||||
directory TEXT BINARY COMMENT 'full path to parent directory',
|
||||
meta LONGBLOB,
|
||||
PRIMARY KEY (dirhash, name)
|
||||
) DEFAULT CHARSET=utf8;
|
||||
```
|
||||
|
||||
s3/filer/master needs the label **sw-backend=true**
|
||||
Alternative database can also be configured (e.g. leveldb) following the instructions at `filer.extraEnvironmentVars`.
|
||||
|
||||
volume need the label **sw-volume=true**
|
||||
### Node Labels
|
||||
Kubernetes node have labels which help to define which node(Host) will run which pod:
|
||||
* s3/filer/master needs the label **sw-backend=true**
|
||||
* volume need the label **sw-volume=true**
|
||||
|
||||
to label a node to be able to run all pod types in k8s:
|
||||
```
|
||||
@@ -29,7 +43,7 @@ please set/update the corresponding affinity rule in values.yaml to an empty one
|
||||
|
||||
```affinity: ""```
|
||||
|
||||
### PVC - storage class ###
|
||||
## PVC - storage class ###
|
||||
|
||||
on the volume stateful set added support for K8S PVC, currently example
|
||||
with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
|
||||
@@ -38,7 +52,7 @@ https://github.com/rancher/local-path-provisioner
|
||||
you can use ANY storage class you like, just update the correct storage-class
|
||||
for your deployment.
|
||||
|
||||
### current instances config (AIO):
|
||||
## current instances config (AIO):
|
||||
1 instance for each type (master/filer+s3/volume)
|
||||
|
||||
you can update the replicas count for each node type in values.yaml,
|
||||
|
||||
@@ -1,14 +1,5 @@
|
||||
{
|
||||
"__inputs": [
|
||||
{
|
||||
"name": "DS_PROMETHEUS-DEV",
|
||||
"label": "prometheus-dev",
|
||||
"description": "",
|
||||
"type": "datasource",
|
||||
"pluginId": "prometheus",
|
||||
"pluginName": "Prometheus"
|
||||
}
|
||||
],
|
||||
"__inputs": [],
|
||||
"__requires": [
|
||||
{
|
||||
"type": "grafana",
|
||||
@@ -33,7 +24,7 @@
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
@@ -61,7 +52,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
@@ -151,7 +142,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
@@ -247,7 +238,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
@@ -343,7 +334,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 0,
|
||||
@@ -448,7 +439,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
@@ -538,7 +529,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
@@ -628,7 +619,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
@@ -718,7 +709,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 0,
|
||||
@@ -811,7 +802,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 0,
|
||||
@@ -943,7 +934,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
@@ -1031,7 +1022,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
@@ -1117,7 +1108,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"fill": 1,
|
||||
"id": 48,
|
||||
"legend": {
|
||||
@@ -1200,7 +1191,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"fill": 1,
|
||||
"id": 50,
|
||||
"legend": {
|
||||
@@ -1283,7 +1274,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"fill": 1,
|
||||
"id": 51,
|
||||
"legend": {
|
||||
@@ -1371,7 +1362,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
@@ -1451,7 +1442,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
@@ -1546,7 +1537,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
@@ -1651,7 +1642,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
@@ -1731,7 +1722,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"editable": true,
|
||||
"error": false,
|
||||
"fill": 1,
|
||||
@@ -1819,7 +1810,29 @@
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": []
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"selected": true,
|
||||
"text": "Prometheus",
|
||||
"value": "Prometheus"
|
||||
},
|
||||
"description": null,
|
||||
"error": null,
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Datasource",
|
||||
"multi": false,
|
||||
"name": "DS_PROMETHEUS",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"queryValue": "",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"type": "datasource"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-30d",
|
||||
|
||||
@@ -113,7 +113,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any PVC exists */}}
|
||||
{{/* check if any Volume PVC exists */}}
|
||||
{{- define "volume.pvc_exists" -}}
|
||||
{{- if or (or (eq .Values.volume.data.type "persistentVolumeClaim") (and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
@@ -122,7 +122,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any HostPath exists */}}
|
||||
{{/* check if any Volume HostPath exists */}}
|
||||
{{- define "volume.hostpath_exists" -}}
|
||||
{{- if or (or (eq .Values.volume.data.type "hostPath") (and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "hostPath") -}}
|
||||
{{- printf "true" -}}
|
||||
@@ -134,3 +134,52 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Filer PVC exists */}}
|
||||
{{- define "filer.pvc_exists" -}}
|
||||
{{- if or (eq .Values.filer.data.type "persistentVolumeClaim") (eq .Values.filer.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
{{- printf "false" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Filer HostPath exists */}}
|
||||
{{- define "filer.hostpath_exists" -}}
|
||||
{{- if or (eq .Values.filer.data.type "hostPath") (eq .Values.filer.logs.type "hostPath") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
{{- printf "false" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Master PVC exists */}}
|
||||
{{- define "master.pvc_exists" -}}
|
||||
{{- if or (eq .Values.master.data.type "persistentVolumeClaim") (eq .Values.master.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
{{- printf "false" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Master HostPath exists */}}
|
||||
{{- define "master.hostpath_exists" -}}
|
||||
{{- if or (eq .Values.master.data.type "hostPath") (eq .Values.master.logs.type "hostPath") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
{{- if or .Values.global.enableSecurity .Values.volume.extraVolumes -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
{{- printf "false" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any InitContainers exist for Volumes */}}
|
||||
{{- define "volume.initContainers_exists" -}}
|
||||
{{- if or (not (empty .Values.volume.dir_idx )) (not (empty .Values.volume.initContainers )) -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
{{- printf "false" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
@@ -20,6 +20,12 @@ spec:
|
||||
port: {{ .Values.filer.grpcPort }}
|
||||
targetPort: {{ .Values.filer.grpcPort }}
|
||||
protocol: TCP
|
||||
{{- if .Values.filer.metricsPort }}
|
||||
- name: "metrics"
|
||||
port: {{ .Values.filer.metricsPort }}
|
||||
targetPort: {{ .Values.filer.metricsPort }}
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
selector:
|
||||
app: {{ template "seaweedfs.name" . }}
|
||||
component: filer
|
||||
@@ -11,10 +11,11 @@ metadata:
|
||||
spec:
|
||||
endpoints:
|
||||
- interval: 30s
|
||||
port: swfs-filer-metrics
|
||||
port: metrics
|
||||
scrapeTimeout: 5s
|
||||
selector:
|
||||
app: {{ template "seaweedfs.name" . }}
|
||||
component: filer
|
||||
matchLabels:
|
||||
app: {{ template "seaweedfs.name" . }}
|
||||
component: filer
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -46,12 +46,16 @@ spec:
|
||||
imagePullSecrets:
|
||||
- name: {{ .Values.global.imagePullSecrets }}
|
||||
{{- end }}
|
||||
serviceAccountName: seaweefds-rw-sa #hack for delete pod master after migration
|
||||
serviceAccountName: seaweedfs-rw-sa #hack for delete pod master after migration
|
||||
terminationGracePeriodSeconds: 60
|
||||
{{- if .Values.filer.priorityClassName }}
|
||||
priorityClassName: {{ .Values.filer.priorityClassName | quote }}
|
||||
{{- end }}
|
||||
enableServiceLinks: false
|
||||
{{- if .Values.filer.initContainers }}
|
||||
initContainers:
|
||||
{{ tpl .Values.filer.initContainers . | nindent 8 | trim }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "filer.image" . }}
|
||||
@@ -106,7 +110,7 @@ spec:
|
||||
filer \
|
||||
-port={{ .Values.filer.port }} \
|
||||
{{- if .Values.filer.metricsPort }}
|
||||
-metricsPort {{ .Values.filer.metricsPort }} \
|
||||
-metricsPort={{ .Values.filer.metricsPort }} \
|
||||
{{- end }}
|
||||
{{- if .Values.filer.redirectOnRead }}
|
||||
-redirectOnRead \
|
||||
@@ -189,6 +193,8 @@ spec:
|
||||
ports:
|
||||
- containerPort: {{ .Values.filer.port }}
|
||||
name: swfs-filer
|
||||
- containerPort: {{ .Values.filer.metricsPort }}
|
||||
name: metrics
|
||||
- containerPort: {{ .Values.filer.grpcPort }}
|
||||
#name: swfs-filer-grpc
|
||||
readinessProbe:
|
||||
@@ -216,10 +222,18 @@ spec:
|
||||
{{ tpl .Values.filer.resources . | nindent 12 | trim }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.filer.logs.type "hostPath" }}
|
||||
- name: seaweedfs-filer-log-volume
|
||||
hostPath:
|
||||
path: /storage/logs/seaweedfs/filer
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- if eq .Values.filer.data.type "hostPath" }}
|
||||
- name: data-filer
|
||||
hostPath:
|
||||
path: /storage/filer_store
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
- name: db-schema-config-volume
|
||||
configMap:
|
||||
name: seaweedfs-db-init-config
|
||||
@@ -253,6 +267,7 @@ spec:
|
||||
{{ tpl .Values.filer.nodeSelector . | indent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.enablePVC }}
|
||||
# DEPRECATION: Deprecate in favor of filer.data section below
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: data-filer
|
||||
@@ -266,4 +281,28 @@ spec:
|
||||
storageClassName: {{ .Values.filer.storageClass }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- $pvc_exists := include "filer.pvc_exists" . -}}
|
||||
{{- if $pvc_exists }}
|
||||
volumeClaimTemplates:
|
||||
{{- if eq .Values.filer.data.type "persistentVolumeClaim"}}
|
||||
- metadata:
|
||||
name: data-filer
|
||||
spec:
|
||||
accessModes: [ "ReadWriteOnce" ]
|
||||
storageClassName: {{ .Values.filer.data.storageClass }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.filer.data.size }}
|
||||
{{- end }}
|
||||
{{- if eq .Values.filer.logs.type "persistentVolumeClaim"}}
|
||||
- metadata:
|
||||
name: seaweedfs-filer-log-volume
|
||||
spec:
|
||||
accessModes: [ "ReadWriteOnce" ]
|
||||
storageClassName: {{ .Values.filer.logs.storageClass }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.filer.logs.size }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -20,6 +20,12 @@ spec:
|
||||
port: {{ .Values.master.grpcPort }}
|
||||
targetPort: {{ .Values.master.grpcPort }}
|
||||
protocol: TCP
|
||||
{{- if .Values.master.metricsPort }}
|
||||
- name: "metrics"
|
||||
port: {{ .Values.master.metricsPort }}
|
||||
targetPort: {{ .Values.master.metricsPort }}
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
selector:
|
||||
app: {{ template "seaweedfs.name" . }}
|
||||
component: master
|
||||
@@ -0,0 +1,21 @@
|
||||
{{- if .Values.master.metricsPort }}
|
||||
{{- if .Values.global.monitoring.enabled }}
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: {{ template "seaweedfs.name" . }}-master
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app: {{ template "seaweedfs.name" . }}
|
||||
component: master
|
||||
spec:
|
||||
endpoints:
|
||||
- interval: 30s
|
||||
port: metrics
|
||||
scrapeTimeout: 5s
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ template "seaweedfs.name" . }}
|
||||
component: master
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -51,6 +51,10 @@ spec:
|
||||
priorityClassName: {{ .Values.master.priorityClassName | quote }}
|
||||
{{- end }}
|
||||
enableServiceLinks: false
|
||||
{{- if .Values.master.initContainers }}
|
||||
initContainers:
|
||||
{{ tpl .Values.master.initContainers . | nindent 8 | trim }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "master.image" . }}
|
||||
@@ -105,7 +109,14 @@ spec:
|
||||
-volumePreallocate \
|
||||
{{- end }}
|
||||
{{- if .Values.global.monitoring.enabled }}
|
||||
{{- if and .Values.global.monitoring.gatewayHost .Values.global.monitoring.gatewayPort }}
|
||||
-metrics.address="{{ .Values.global.monitoring.gatewayHost }}:{{ .Values.global.monitoring.gatewayPort }}" \
|
||||
{{- if .Values.master.metricsIntervalSec }}
|
||||
-metrics.intervalSeconds={{ .Values.master.metricsIntervalSec }} \
|
||||
{{- end }}
|
||||
{{- else if .Values.master.metricsPort }}
|
||||
-metricsPort={{ .Values.master.metricsPort }} \
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
-volumeSizeLimitMB={{ .Values.master.volumeSizeLimitMB }} \
|
||||
{{- if .Values.master.disableHttp }}
|
||||
@@ -117,9 +128,6 @@ spec:
|
||||
{{- if .Values.master.garbageThreshold }}
|
||||
-garbageThreshold={{ .Values.master.garbageThreshold }} \
|
||||
{{- end }}
|
||||
{{- if .Values.master.metricsIntervalSec }}
|
||||
-metrics.intervalSeconds={{ .Values.master.metricsIntervalSec }} \
|
||||
{{- end }}
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master \
|
||||
-peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
volumeMounts:
|
||||
@@ -152,6 +160,10 @@ spec:
|
||||
ports:
|
||||
- containerPort: {{ .Values.master.port }}
|
||||
name: swfs-master
|
||||
{{- if and .Values.global.monitoring.enabled .Values.master.metricsPort }}
|
||||
- containerPort: {{ .Values.master.metricsPort }}
|
||||
name: metrics
|
||||
{{- end }}
|
||||
- containerPort: {{ .Values.master.grpcPort }}
|
||||
#name: swfs-master-grpc
|
||||
readinessProbe:
|
||||
@@ -178,15 +190,21 @@ spec:
|
||||
resources:
|
||||
{{ tpl .Values.master.resources . | nindent 12 | trim }}
|
||||
{{- end }}
|
||||
{{- $hostpath_exists := include "master.hostpath_exists" . -}}
|
||||
{{- if $hostpath_exists }}
|
||||
volumes:
|
||||
{{- if eq .Values.master.logs.type "hostPath" }}
|
||||
- name: seaweedfs-master-log-volume
|
||||
hostPath:
|
||||
path: /storage/logs/seaweedfs/master
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- if eq .Values.master.data.type "hostPath" }}
|
||||
- name: data-{{ .Release.Namespace }}
|
||||
hostPath:
|
||||
path: /ssd/seaweed-master/
|
||||
type: DirectoryOrCreate
|
||||
{{- end }}
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
@@ -208,20 +226,33 @@ spec:
|
||||
secretName: {{ template "seaweedfs.name" . }}-client-cert
|
||||
{{- end }}
|
||||
{{ tpl .Values.master.extraVolumes . | indent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- if .Values.master.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{ tpl .Values.master.nodeSelector . | indent 8 | trim }}
|
||||
{{- end }}
|
||||
{{/* volumeClaimTemplates:*/}}
|
||||
{{/* - metadata:*/}}
|
||||
{{/* name: data-{{ .Release.Namespace }}*/}}
|
||||
{{/* spec:*/}}
|
||||
{{/* accessModes:*/}}
|
||||
{{/* - ReadWriteOnce*/}}
|
||||
{{/* resources:*/}}
|
||||
{{/* requests:*/}}
|
||||
{{/* storage: {{ .Values.master.storage }}*/}}
|
||||
{{/* {{- if .Values.master.storageClass }}*/}}
|
||||
{{/* storageClassName: {{ .Values.master.storageClass }}*/}}
|
||||
{{/* {{- end }}*/}}
|
||||
{{- $pvc_exists := include "volume.pvc_exists" . -}}
|
||||
{{- if $pvc_exists }}
|
||||
volumeClaimTemplates:
|
||||
{{- if eq .Values.master.data.type "persistentVolumeClaim"}}
|
||||
- metadata:
|
||||
name: data-{{ .Release.Namespace }}
|
||||
spec:
|
||||
accessModes: [ "ReadWriteOnce" ]
|
||||
storageClassName: {{ .Values.master.data.storageClass }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.master.data.size }}
|
||||
{{- end }}
|
||||
{{- if eq .Values.master.logs.type "persistentVolumeClaim"}}
|
||||
- metadata:
|
||||
name: seaweedfs-master-log-volume
|
||||
spec:
|
||||
accessModes: [ "ReadWriteOnce" ]
|
||||
storageClassName: {{ .Values.master.logs.storageClass }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.master.logs.size }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -39,6 +39,10 @@ spec:
|
||||
priorityClassName: {{ .Values.s3.priorityClassName | quote }}
|
||||
{{- end }}
|
||||
enableServiceLinks: false
|
||||
{{- if .Values.s3.initContainers }}
|
||||
initContainers:
|
||||
{{ tpl .Values.s3.initContainers . | nindent 8 | trim }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "s3.image" . }}
|
||||
@@ -127,6 +131,10 @@ spec:
|
||||
ports:
|
||||
- containerPort: {{ .Values.s3.port }}
|
||||
name: swfs-s3
|
||||
{{- if .Values.s3.metricsPort }}
|
||||
- containerPort: {{ .Values.s3.metricsPort }}
|
||||
name: "metrics"
|
||||
{{- end }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /status
|
||||
|
||||
@@ -11,10 +11,11 @@ metadata:
|
||||
spec:
|
||||
endpoints:
|
||||
- interval: 30s
|
||||
port: swfs-s3-metrics
|
||||
port: metrics
|
||||
scrapeTimeout: 5s
|
||||
selector:
|
||||
app: {{ template "seaweedfs.name" . }}
|
||||
component: s3
|
||||
matchLabels:
|
||||
app: {{ template "seaweedfs.name" . }}
|
||||
component: s3
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -3,7 +3,7 @@
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: seaweefds-rw-cr
|
||||
name: seaweedfs-rw-cr
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
@@ -12,18 +12,18 @@ rules:
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: seaweefds-rw-sa
|
||||
name: seaweedfs-rw-sa
|
||||
namespace: {{ .Release.Namespace }}
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: system:serviceaccount:seaweefds-rw-sa:default
|
||||
name: system:serviceaccount:seaweedfs-rw-sa:default
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: seaweefds-rw-sa
|
||||
name: seaweedfs-rw-sa
|
||||
namespace: {{ .Release.Namespace }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: seaweefds-rw-cr
|
||||
name: seaweedfs-rw-cr
|
||||
@@ -18,7 +18,7 @@ spec:
|
||||
targetPort: {{ .Values.volume.grpcPort }}
|
||||
protocol: TCP
|
||||
{{- if .Values.volume.metricsPort }}
|
||||
- name: "swfs-volume-metrics"
|
||||
- name: "metrics"
|
||||
port: {{ .Values.volume.metricsPort }}
|
||||
targetPort: {{ .Values.volume.metricsPort }}
|
||||
protocol: TCP
|
||||
|
||||
@@ -11,10 +11,11 @@ metadata:
|
||||
spec:
|
||||
endpoints:
|
||||
- interval: 30s
|
||||
port: swfs-volume-metrics
|
||||
port: metrics
|
||||
scrapeTimeout: 5s
|
||||
selector:
|
||||
app: {{ template "seaweedfs.name" . }}
|
||||
component: volume
|
||||
matchLabels:
|
||||
app: {{ template "seaweedfs.name" . }}
|
||||
component: volume
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -45,18 +45,24 @@ spec:
|
||||
priorityClassName: {{ .Values.volume.priorityClassName | quote }}
|
||||
{{- end }}
|
||||
enableServiceLinks: false
|
||||
{{- if .Values.volume.dir_idx }}
|
||||
{{- $initContainers_exists := include "volume.initContainers_exists" . -}}
|
||||
{{- if $initContainers_exists }}
|
||||
initContainers:
|
||||
{{- if .Values.volume.dir_idx }}
|
||||
- name: seaweedfs-vol-move-idx
|
||||
image: {{ template "volume.image" . }}
|
||||
imagePullPolicy: {{ .Values.global.pullPolicy | default "IfNotPresent" }}
|
||||
imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }}
|
||||
command: [ '/bin/sh', '-c' ]
|
||||
args: ['if ls {{ .Values.volume.dir }}/*.idx >/dev/null 2>&1; then mv {{ .Values.volume.dir }}/*.idx {{ .Values.volume.dir_idx }}/; fi;']
|
||||
args: [ 'if ls {{ .Values.volume.dir }}/*.idx >/dev/null 2>&1; then mv {{ .Values.volume.dir }}/*.idx {{ .Values.volume.dir_idx }}/; fi;' ]
|
||||
volumeMounts:
|
||||
- name: idx
|
||||
mountPath: {{ .Values.volume.dir_idx }}
|
||||
- name: data
|
||||
mountPath: {{ .Values.volume.dir }}
|
||||
- name: idx
|
||||
mountPath: {{ .Values.volume.dir_idx }}
|
||||
- name: data
|
||||
mountPath: {{ .Values.volume.dir }}
|
||||
{{- end }}
|
||||
{{- if .Values.volume.initContainers }}
|
||||
{{ tpl .Values.volume.initContainers . | nindent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
@@ -96,7 +102,7 @@ spec:
|
||||
volume \
|
||||
-port={{ .Values.volume.port }} \
|
||||
{{- if .Values.volume.metricsPort }}
|
||||
-metricsPort {{ .Values.volume.metricsPort }} \
|
||||
-metricsPort={{ .Values.volume.metricsPort }} \
|
||||
{{- end }}
|
||||
-dir={{ .Values.volume.dir }} \
|
||||
{{- if .Values.volume.dir_idx }}
|
||||
@@ -164,8 +170,12 @@ spec:
|
||||
ports:
|
||||
- containerPort: {{ .Values.volume.port }}
|
||||
name: swfs-vol
|
||||
{{- if .Values.volume.metricsPort }}
|
||||
- containerPort: {{ .Values.filer.metricsPort }}
|
||||
name: metrics
|
||||
{{- end }}
|
||||
- containerPort: {{ .Values.volume.grpcPort }}
|
||||
#name: swfs-vol-grpc
|
||||
name: swfs-vol-grpc
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /status
|
||||
|
||||
@@ -39,6 +39,7 @@ master:
|
||||
replicas: 1
|
||||
port: 9333
|
||||
grpcPort: 19333
|
||||
metricsPort: 9327
|
||||
ipBind: "0.0.0.0"
|
||||
volumePreallocate: false
|
||||
volumeSizeLimitMB: 1000
|
||||
@@ -58,16 +59,26 @@ master:
|
||||
# Disable http request, only gRpc operations are allowed
|
||||
disableHttp: false
|
||||
|
||||
# can use ANY storage-class , example with local-path-provisioner
|
||||
# data:
|
||||
# type: "persistentVolumeClaim"
|
||||
# size: "24Ti"
|
||||
# storageClass: "local-path-provisioner"
|
||||
data:
|
||||
type: "hostPath"
|
||||
size: ""
|
||||
storageClass: ""
|
||||
|
||||
logs:
|
||||
type: "hostPath"
|
||||
size: ""
|
||||
storageClass: ""
|
||||
|
||||
initContainers: ""
|
||||
|
||||
extraVolumes: ""
|
||||
extraVolumeMounts: ""
|
||||
|
||||
# storage and storageClass are the settings for configuring stateful
|
||||
# storage for the master pods. storage should be set to the disk size of
|
||||
# the attached volume. storageClass is the class of storage which defaults
|
||||
# to null (the Kube cluster will pick the default).
|
||||
storage: 25Gi
|
||||
storageClass: null
|
||||
|
||||
# Resource requests, limits, etc. for the master cluster placement. This
|
||||
# should map directly to the value of the resources field for a PodSpec,
|
||||
# formatted as a multi-line string. By default no direct resource request
|
||||
@@ -156,7 +167,7 @@ volume:
|
||||
# minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly
|
||||
minFreeSpacePercent: 7
|
||||
|
||||
# can use ANY storage-class , example with local-path-provisner
|
||||
# can use ANY storage-class , example with local-path-provisioner
|
||||
# data:
|
||||
# type: "persistentVolumeClaim"
|
||||
# size: "24Ti"
|
||||
@@ -202,6 +213,8 @@ volume:
|
||||
# Adjust jpg orientation when uploading.
|
||||
imagesFixOrientation: false
|
||||
|
||||
initContainers: ""
|
||||
|
||||
extraVolumes: ""
|
||||
extraVolumeMounts: ""
|
||||
|
||||
@@ -275,16 +288,34 @@ filer:
|
||||
# Disable http request, only gRpc operations are allowed
|
||||
disableHttp: false
|
||||
|
||||
# DEPRECATE: enablePVC, storage, storageClass
|
||||
# Consider replacing with filer.data section below instead.
|
||||
|
||||
# Settings for configuring stateful storage of filer pods.
|
||||
# enablePVC will create a pvc for filer for data persistence.
|
||||
enablePVC: false
|
||||
|
||||
# storage and storageClass are the settings for configuring stateful
|
||||
# storage for the master pods. storage should be set to the disk size of
|
||||
# the attached volume. storageClass is the class of storage which defaults
|
||||
# to null (the Kube cluster will pick the default).
|
||||
# storage should be set to the disk size of the attached volume.
|
||||
storage: 25Gi
|
||||
# storageClass is the class of storage which defaults to null (the Kube cluster will pick the default).
|
||||
storageClass: null
|
||||
|
||||
# can use ANY storage-class , example with local-path-provisioner
|
||||
# data:
|
||||
# type: "persistentVolumeClaim"
|
||||
# size: "24Ti"
|
||||
# storageClass: "local-path-provisioner"
|
||||
data:
|
||||
type: "hostPath"
|
||||
size: ""
|
||||
storageClass: ""
|
||||
|
||||
logs:
|
||||
type: "hostPath"
|
||||
size: ""
|
||||
storageClass: ""
|
||||
|
||||
initContainers: ""
|
||||
|
||||
extraVolumes: ""
|
||||
extraVolumeMounts: ""
|
||||
|
||||
@@ -400,6 +431,8 @@ s3:
|
||||
# Suffix of the host name, {bucket}.{domainName}
|
||||
domainName: ""
|
||||
|
||||
initContainers: ""
|
||||
|
||||
extraVolumes: ""
|
||||
extraVolumeMounts: ""
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 136 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 7.2 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 12 KiB |
@@ -5,7 +5,7 @@
|
||||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>3.13</version>
|
||||
<version>3.30</version>
|
||||
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
@@ -14,7 +14,7 @@
|
||||
</parent>
|
||||
|
||||
<properties>
|
||||
<protobuf.version>3.16.1</protobuf.version>
|
||||
<protobuf.version>3.16.3</protobuf.version>
|
||||
<!-- follow https://github.com/grpc/grpc-java -->
|
||||
<grpc.version>1.23.0</grpc.version>
|
||||
<guava.version>30.0-jre</guava.version>
|
||||
@@ -159,7 +159,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<version>3.0.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>3.13</version>
|
||||
<version>3.30</version>
|
||||
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
@@ -154,7 +154,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<version>3.0.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>3.13</version>
|
||||
<version>3.30</version>
|
||||
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
@@ -128,7 +128,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<version>3.0.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
|
||||
@@ -14,7 +14,9 @@ import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.security.SecureRandom;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.List;
|
||||
import java.util.Base64;
|
||||
|
||||
public class SeaweedWrite {
|
||||
|
||||
@@ -123,13 +125,20 @@ public class SeaweedWrite {
|
||||
final byte[] bytes,
|
||||
final long bytesOffset, final long bytesLength,
|
||||
byte[] cipherKey) throws IOException {
|
||||
MessageDigest md = null;
|
||||
try {
|
||||
md = MessageDigest.getInstance("MD5");
|
||||
} catch (java.security.NoSuchAlgorithmException e) {
|
||||
}
|
||||
|
||||
InputStream inputStream = null;
|
||||
if (cipherKey == null || cipherKey.length == 0) {
|
||||
md.update(bytes, (int) bytesOffset, (int) bytesLength);
|
||||
inputStream = new ByteArrayInputStream(bytes, (int) bytesOffset, (int) bytesLength);
|
||||
} else {
|
||||
try {
|
||||
byte[] encryptedBytes = SeaweedCipher.encrypt(bytes, (int) bytesOffset, (int) bytesLength, cipherKey);
|
||||
md.update(encryptedBytes);
|
||||
inputStream = new ByteArrayInputStream(encryptedBytes, 0, encryptedBytes.length);
|
||||
} catch (Exception e) {
|
||||
throw new IOException("fail to encrypt data", e);
|
||||
@@ -140,6 +149,7 @@ public class SeaweedWrite {
|
||||
if (auth != null && auth.length() != 0) {
|
||||
post.addHeader("Authorization", "BEARER " + auth);
|
||||
}
|
||||
post.addHeader("Content-MD5", Base64.getEncoder().encodeToString(md.digest()));
|
||||
|
||||
post.setEntity(MultipartEntityBuilder.create()
|
||||
.setMode(HttpMultipartMode.BROWSER_COMPATIBLE)
|
||||
@@ -149,6 +159,13 @@ public class SeaweedWrite {
|
||||
CloseableHttpResponse response = SeaweedUtil.getClosableHttpClient().execute(post);
|
||||
|
||||
try {
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
if (response.getEntity().getContentType() != null && response.getEntity().getContentType().getValue().equals("application/json")) {
|
||||
throw new IOException(EntityUtils.toString(response.getEntity(), "UTF-8"));
|
||||
} else {
|
||||
throw new IOException(response.getStatusLine().getReasonPhrase());
|
||||
}
|
||||
}
|
||||
|
||||
String etag = response.getLastHeader("ETag").getValue();
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ syntax = "proto3";
|
||||
|
||||
package filer_pb;
|
||||
|
||||
option go_package = "github.com/chrislusf/seaweedfs/weed/pb/filer_pb";
|
||||
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb";
|
||||
option java_package = "seaweedfs.client";
|
||||
option java_outer_classname = "FilerProto";
|
||||
|
||||
@@ -60,12 +60,6 @@ service SeaweedFiler {
|
||||
rpc SubscribeLocalMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
|
||||
}
|
||||
|
||||
rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) {
|
||||
}
|
||||
|
||||
rpc LocateBroker (LocateBrokerRequest) returns (LocateBrokerResponse) {
|
||||
}
|
||||
|
||||
rpc KvGet (KvGetRequest) returns (KvGetResponse) {
|
||||
}
|
||||
|
||||
@@ -277,6 +271,7 @@ message Location {
|
||||
string url = 1;
|
||||
string public_url = 2;
|
||||
uint32 grpc_port = 3;
|
||||
string data_center = 4;
|
||||
}
|
||||
message LookupVolumeResponse {
|
||||
map<string, Locations> locations_map = 1;
|
||||
@@ -346,6 +341,8 @@ message SubscribeMetadataRequest {
|
||||
repeated string path_prefixes = 6;
|
||||
int32 client_id = 7;
|
||||
int64 until_ns = 8;
|
||||
int32 client_epoch = 9;
|
||||
repeated string directories = 10; // exact directory to watch
|
||||
}
|
||||
message SubscribeMetadataResponse {
|
||||
string directory = 1;
|
||||
|
||||
@@ -11,13 +11,13 @@
|
||||
<dependency>
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>3.13</version>
|
||||
<version>3.30</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-hadoop2-client</artifactId>
|
||||
<version>3.13</version>
|
||||
<version>3.30</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<version>3.2.3</version>
|
||||
<version>3.2.4</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
|
||||
@@ -108,7 +108,7 @@
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<version>3.0.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
@@ -124,7 +124,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client</artifactId>
|
||||
<version>2.10.1</version>
|
||||
<version>3.2.4</version>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
@@ -132,11 +132,11 @@
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>hadoop-mapreduce-client-app</artifactId>
|
||||
<artifactId>hadoop-yarn-api</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>hadoop-yarn-api</artifactId>
|
||||
<artifactId>hadoop-yarn-client</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
@@ -156,7 +156,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<version>2.10.1</version>
|
||||
<version>3.2.4</version>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
@@ -167,10 +167,6 @@
|
||||
<artifactId>commons-math3</artifactId>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>xmlenc</artifactId>
|
||||
<groupId>xmlenc</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-io</artifactId>
|
||||
<groupId>commons-io</groupId>
|
||||
@@ -184,20 +180,28 @@
|
||||
<groupId>commons-collections</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>servlet-api</artifactId>
|
||||
<artifactId>javax.servlet-api</artifactId>
|
||||
<groupId>javax.servlet</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty</artifactId>
|
||||
<groupId>org.mortbay.jetty</groupId>
|
||||
<artifactId>javax.activation-api</artifactId>
|
||||
<groupId>javax.activation</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty-server</artifactId>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty-util</artifactId>
|
||||
<groupId>org.mortbay.jetty</groupId>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty-sslengine</artifactId>
|
||||
<groupId>org.mortbay.jetty</groupId>
|
||||
<artifactId>jetty-servlet</artifactId>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty-webapp</artifactId>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jsp-api</artifactId>
|
||||
@@ -207,6 +211,10 @@
|
||||
<artifactId>jersey-core</artifactId>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jersey-servlet</artifactId>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jersey-json</artifactId>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
@@ -216,49 +224,37 @@
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>log4j</artifactId>
|
||||
<groupId>log4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jets3t</artifactId>
|
||||
<groupId>net.java.dev.jets3t</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-lang</artifactId>
|
||||
<groupId>commons-lang</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-configuration</artifactId>
|
||||
<groupId>commons-configuration</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-digester</artifactId>
|
||||
<groupId>commons-digester</groupId>
|
||||
<artifactId>reload4j</artifactId>
|
||||
<groupId>ch.qos.reload4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-beanutils</artifactId>
|
||||
<groupId>commons-beanutils</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-configuration2</artifactId>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<artifactId>commons-text</artifactId>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>slf4j-reload4j</artifactId>
|
||||
<groupId>org.slf4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jackson-core-asl</artifactId>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jackson-mapper-asl</artifactId>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>avro</artifactId>
|
||||
<groupId>org.apache.avro</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>re2j</artifactId>
|
||||
<groupId>com.google.re2j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>hadoop-auth</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
@@ -287,6 +283,14 @@
|
||||
<artifactId>commons-compress</artifactId>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>kerb-simplekdc</artifactId>
|
||||
<groupId>org.apache.kerby</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>stax2-api</artifactId>
|
||||
<groupId>org.codehaus.woodstox</groupId>
|
||||
@@ -295,6 +299,10 @@
|
||||
<artifactId>woodstox-core</artifactId>
|
||||
<groupId>com.fasterxml.woodstox</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>dnsjava</artifactId>
|
||||
<groupId>dnsjava</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>hadoop-annotations</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
@@ -309,7 +317,7 @@
|
||||
</snapshotRepository>
|
||||
</distributionManagement>
|
||||
<properties>
|
||||
<seaweedfs.client.version>3.13</seaweedfs.client.version>
|
||||
<hadoop.version>2.10.1</hadoop.version>
|
||||
<seaweedfs.client.version>3.30</seaweedfs.client.version>
|
||||
<hadoop.version>3.2.4</hadoop.version>
|
||||
</properties>
|
||||
</project>
|
||||
@@ -5,8 +5,8 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<properties>
|
||||
<seaweedfs.client.version>3.13</seaweedfs.client.version>
|
||||
<hadoop.version>2.10.1</hadoop.version>
|
||||
<seaweedfs.client.version>3.30</seaweedfs.client.version>
|
||||
<hadoop.version>3.2.4</hadoop.version>
|
||||
</properties>
|
||||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
@@ -129,7 +129,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<version>3.0.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
|
||||
@@ -108,7 +108,7 @@
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<version>3.0.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
@@ -124,7 +124,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client</artifactId>
|
||||
<version>3.2.3</version>
|
||||
<version>3.2.4</version>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
@@ -156,7 +156,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<version>3.2.3</version>
|
||||
<version>3.2.4</version>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
@@ -224,8 +224,8 @@
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>log4j</artifactId>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>reload4j</artifactId>
|
||||
<groupId>ch.qos.reload4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-beanutils</artifactId>
|
||||
@@ -244,7 +244,7 @@
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<artifactId>slf4j-reload4j</artifactId>
|
||||
<groupId>org.slf4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
@@ -317,7 +317,7 @@
|
||||
</snapshotRepository>
|
||||
</distributionManagement>
|
||||
<properties>
|
||||
<seaweedfs.client.version>3.13</seaweedfs.client.version>
|
||||
<hadoop.version>3.2.3</hadoop.version>
|
||||
<seaweedfs.client.version>3.30</seaweedfs.client.version>
|
||||
<hadoop.version>3.2.4</hadoop.version>
|
||||
</properties>
|
||||
</project>
|
||||
@@ -5,8 +5,8 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<properties>
|
||||
<seaweedfs.client.version>3.13</seaweedfs.client.version>
|
||||
<hadoop.version>3.2.3</hadoop.version>
|
||||
<seaweedfs.client.version>3.30</seaweedfs.client.version>
|
||||
<hadoop.version>3.2.4</hadoop.version>
|
||||
</properties>
|
||||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
@@ -129,7 +129,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<version>3.0.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
|
||||
@@ -1891,7 +1891,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "go_memstats_alloc_bytes{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}",
|
||||
"expr": "go_memstats_alloc_bytes{namespace=~\"$namespace\", endpoint=\"metrics\"}",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
@@ -1909,7 +1909,7 @@
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "go_memstats_stack_inuse_bytes{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}",
|
||||
"expr": "go_memstats_stack_inuse_bytes{namespace=~\"$namespace\", endpoint=\"metrics\"}",
|
||||
"format": "time_series",
|
||||
"hide": true,
|
||||
"interval": "",
|
||||
@@ -1918,7 +1918,7 @@
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "go_memstats_heap_inuse_bytes{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}",
|
||||
"expr": "go_memstats_heap_inuse_bytes{namespace=~\"$namespace\", endpoint=\"metrics\"}",
|
||||
"format": "time_series",
|
||||
"hide": true,
|
||||
"interval": "",
|
||||
@@ -2019,7 +2019,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "go_gc_duration_seconds{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}",
|
||||
"expr": "go_gc_duration_seconds{namespace=~\"$namespace\", endpoint=\"metrics\"}",
|
||||
"format": "time_series",
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
@@ -2119,7 +2119,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "go_goroutines{namespace=~\"$namespace\", endpoint=\"swfs-filer-metrics\"}",
|
||||
"expr": "go_goroutines{namespace=~\"$namespace\", endpoint=\"metrics\"}",
|
||||
"format": "time_series",
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
@@ -2203,7 +2203,7 @@
|
||||
"value": "s3"
|
||||
},
|
||||
"datasource": "$DS_PROMETHEUS",
|
||||
"definition": "label_values({endpoint=\"swfs-filer-metrics\"}, namespace)",
|
||||
"definition": "label_values({endpoint=\"metrics\"}, namespace)",
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "namespace",
|
||||
@@ -2216,7 +2216,7 @@
|
||||
"value": "s3"
|
||||
}
|
||||
],
|
||||
"query": "label_values({endpoint=\"swfs-filer-metrics\"}, namespace)",
|
||||
"query": "label_values({endpoint=\"metrics\"}, namespace)",
|
||||
"refresh": 0,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ parts:
|
||||
plugin: go
|
||||
# Snapcraft will look in this location for the source of the application
|
||||
source: .
|
||||
go-importpath: github.com/chrislusf/seaweedfs
|
||||
go-importpath: github.com/seaweedfs/seaweedfs
|
||||
go:
|
||||
# Defines the version of golang which will be bootstrapped into the snap
|
||||
source-tag: go1.14
|
||||
|
||||
@@ -7,10 +7,10 @@ import (
|
||||
"path"
|
||||
"strconv"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/backend"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -6,19 +6,20 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/idx"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/idx"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -155,7 +156,7 @@ func getVolumeFiles(v uint32, addr pb.ServerAddress) (map[types.NeedleId]needleS
|
||||
|
||||
var maxOffset int64
|
||||
files := map[types.NeedleId]needleState{}
|
||||
err = idx.WalkIndexFile(idxFile, func(key types.NeedleId, offset types.Offset, size types.Size) error {
|
||||
err = idx.WalkIndexFile(idxFile, 0, func(key types.NeedleId, offset types.Offset, size types.Size) error {
|
||||
if offset.IsZero() || size.IsDeleted() {
|
||||
files[key] = needleState{
|
||||
state: stateDeleted,
|
||||
|
||||
@@ -8,12 +8,12 @@ import (
|
||||
"path"
|
||||
"strconv"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/backend"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -3,10 +3,11 @@ package main
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
@@ -51,7 +52,7 @@ func main() {
|
||||
}
|
||||
|
||||
func startGenerateMetadata() {
|
||||
pb.WithFilerClient(false, pb.ServerAddress(*tailFiler), grpc.WithInsecure(), func(client filer_pb.SeaweedFilerClient) error {
|
||||
pb.WithFilerClient(false, pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
||||
for i := 0; i < *n; i++ {
|
||||
name := fmt.Sprintf("file%d", i)
|
||||
@@ -77,7 +78,7 @@ func startGenerateMetadata() {
|
||||
|
||||
func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
|
||||
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithInsecure(), "tail", 0, *dir, nil, 0, 0, 0, eachEntryFunc, pb.TrivialOnError)
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), "tail", 0, 0, *dir, nil, 0, 0, 0, eachEntryFunc, pb.TrivialOnError)
|
||||
|
||||
if tailErr != nil {
|
||||
fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)
|
||||
|
||||
@@ -6,11 +6,11 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/backend"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -3,16 +3,16 @@ package main
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"log"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"crypto/md5"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// Downloads an item from an S3 Bucket in the region configured in the shared config
|
||||
@@ -63,6 +64,7 @@ func main() {
|
||||
fmt.Printf("error put request: %v\n", err)
|
||||
return
|
||||
}
|
||||
defer util.CloseResponse(resp)
|
||||
fmt.Printf("response: %+v\n", resp)
|
||||
}
|
||||
|
||||
@@ -70,4 +72,4 @@ var stringContent = `Generate a Pre-Signed URL for an Amazon S3 PUT Operation wi
|
||||
You can generate a pre-signed URL for a PUT operation that checks whether users upload the correct content. When the SDK pre-signs a request, it computes the checksum of the request body and generates an MD5 checksum that is included in the pre-signed URL. Users must upload the same content that produces the same MD5 checksum generated by the SDK; otherwise, the operation fails. This is not the Content-MD5, but the signature. To enforce Content-MD5, simply add the header to the request.
|
||||
|
||||
The following example adds a Body field to generate a pre-signed PUT operation that requires a specific payload to be uploaded by users.
|
||||
`
|
||||
`
|
||||
@@ -2,13 +2,13 @@ package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -3,14 +3,15 @@ package main
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/idx"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/idx"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -36,7 +37,7 @@ func main() {
|
||||
}
|
||||
defer indexFile.Close()
|
||||
|
||||
idx.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size types.Size) error {
|
||||
idx.WalkIndexFile(indexFile, 0, func(key types.NeedleId, offset types.Offset, size types.Size) error {
|
||||
fmt.Printf("key:%v offset:%v size:%v(%v)\n", key, offset, size, util.BytesToHumanReadable(uint64(size)))
|
||||
return nil
|
||||
})
|
||||
|
||||
@@ -7,15 +7,15 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
logdataFile = flag.String("logdata", "", "log data file saved under "+ filer.SystemLogDir)
|
||||
logdataFile = flag.String("logdata", "", "log data file saved under "+filer.SystemLogDir)
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
@@ -7,10 +7,10 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -5,11 +5,11 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
"io"
|
||||
)
|
||||
|
||||
@@ -2,14 +2,14 @@ package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
util2 "github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
util2 "github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"golang.org/x/tools/godoc/util"
|
||||
)
|
||||
|
||||
@@ -38,7 +38,7 @@ func main() {
|
||||
sinceTimeNs = time.Now().Add(-*rewindDuration).UnixNano()
|
||||
}
|
||||
|
||||
err := operation.TailVolume(func()pb.ServerAddress{return pb.ServerAddress(*master)}, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) {
|
||||
err := operation.TailVolume(func() pb.ServerAddress { return pb.ServerAddress(*master) }, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) {
|
||||
if n.Size == 0 {
|
||||
println("-", n.String())
|
||||
return nil
|
||||
|
||||
@@ -52,3 +52,7 @@ debug_filer_remote_sync_buckets:
|
||||
debug_master_follower:
|
||||
go build -gcflags="all=-N -l"
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 master.follower
|
||||
|
||||
debug_filer_sync:
|
||||
go build -gcflags="all=-N -l"
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.sync -a 192.168.2.7:8888 -b 192.168.2.7:8889 -isActivePassive -b.debug
|
||||
+167
-129
@@ -1,8 +1,8 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -15,88 +15,156 @@ const (
|
||||
BrokerType = "broker"
|
||||
)
|
||||
|
||||
type FilerGroup string
|
||||
type Filers struct {
|
||||
filers map[pb.ServerAddress]*ClusterNode
|
||||
leaders *Leaders
|
||||
}
|
||||
type FilerGroupName string
|
||||
type DataCenter string
|
||||
type Rack string
|
||||
|
||||
type Leaders struct {
|
||||
leaders [3]pb.ServerAddress
|
||||
}
|
||||
|
||||
type ClusterNode struct {
|
||||
Address pb.ServerAddress
|
||||
Version string
|
||||
counter int
|
||||
CreatedTs time.Time
|
||||
Address pb.ServerAddress
|
||||
Version string
|
||||
counter int
|
||||
CreatedTs time.Time
|
||||
DataCenter DataCenter
|
||||
Rack Rack
|
||||
}
|
||||
type GroupMembers struct {
|
||||
members map[pb.ServerAddress]*ClusterNode
|
||||
leaders *Leaders
|
||||
}
|
||||
type ClusterNodeGroups struct {
|
||||
groupMembers map[FilerGroupName]*GroupMembers
|
||||
sync.RWMutex
|
||||
}
|
||||
type Cluster struct {
|
||||
filerGroups *ClusterNodeGroups
|
||||
brokerGroups *ClusterNodeGroups
|
||||
}
|
||||
|
||||
type Cluster struct {
|
||||
filerGroup2filers map[FilerGroup]*Filers
|
||||
filersLock sync.RWMutex
|
||||
brokers map[pb.ServerAddress]*ClusterNode
|
||||
brokersLock sync.RWMutex
|
||||
func newClusterNodeGroups() *ClusterNodeGroups {
|
||||
return &ClusterNodeGroups{
|
||||
groupMembers: map[FilerGroupName]*GroupMembers{},
|
||||
}
|
||||
}
|
||||
func (g *ClusterNodeGroups) getGroupMembers(filerGroup FilerGroupName, createIfNotFound bool) *GroupMembers {
|
||||
members, found := g.groupMembers[filerGroup]
|
||||
if !found && createIfNotFound {
|
||||
members = &GroupMembers{
|
||||
members: make(map[pb.ServerAddress]*ClusterNode),
|
||||
leaders: &Leaders{},
|
||||
}
|
||||
g.groupMembers[filerGroup] = members
|
||||
}
|
||||
return members
|
||||
}
|
||||
|
||||
func (m *GroupMembers) addMember(dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) *ClusterNode {
|
||||
if existingNode, found := m.members[address]; found {
|
||||
existingNode.counter++
|
||||
return nil
|
||||
}
|
||||
t := &ClusterNode{
|
||||
Address: address,
|
||||
Version: version,
|
||||
counter: 1,
|
||||
CreatedTs: time.Now(),
|
||||
DataCenter: dataCenter,
|
||||
Rack: rack,
|
||||
}
|
||||
m.members[address] = t
|
||||
return t
|
||||
}
|
||||
func (m *GroupMembers) removeMember(address pb.ServerAddress) bool {
|
||||
if existingNode, found := m.members[address]; !found {
|
||||
return false
|
||||
} else {
|
||||
existingNode.counter--
|
||||
if existingNode.counter <= 0 {
|
||||
delete(m.members, address)
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (g *ClusterNodeGroups) AddClusterNode(filerGroup FilerGroupName, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
|
||||
g.Lock()
|
||||
defer g.Unlock()
|
||||
m := g.getGroupMembers(filerGroup, true)
|
||||
if t := m.addMember(dataCenter, rack, address, version); t != nil {
|
||||
return ensureGroupLeaders(m, true, filerGroup, nodeType, address)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (g *ClusterNodeGroups) RemoveClusterNode(filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
|
||||
g.Lock()
|
||||
defer g.Unlock()
|
||||
m := g.getGroupMembers(filerGroup, false)
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
if m.removeMember(address) {
|
||||
return ensureGroupLeaders(m, false, filerGroup, nodeType, address)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (g *ClusterNodeGroups) ListClusterNode(filerGroup FilerGroupName) (nodes []*ClusterNode) {
|
||||
g.Lock()
|
||||
defer g.Unlock()
|
||||
m := g.getGroupMembers(filerGroup, false)
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
for _, node := range m.members {
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
return
|
||||
}
|
||||
func (g *ClusterNodeGroups) IsOneLeader(filerGroup FilerGroupName, address pb.ServerAddress) bool {
|
||||
g.Lock()
|
||||
defer g.Unlock()
|
||||
m := g.getGroupMembers(filerGroup, false)
|
||||
if m == nil {
|
||||
return false
|
||||
}
|
||||
return m.leaders.isOneLeader(address)
|
||||
}
|
||||
func (g *ClusterNodeGroups) ListClusterNodeLeaders(filerGroup FilerGroupName) (nodes []pb.ServerAddress) {
|
||||
g.Lock()
|
||||
defer g.Unlock()
|
||||
m := g.getGroupMembers(filerGroup, false)
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
return m.leaders.GetLeaders()
|
||||
}
|
||||
|
||||
func NewCluster() *Cluster {
|
||||
return &Cluster{
|
||||
filerGroup2filers: make(map[FilerGroup]*Filers),
|
||||
brokers: make(map[pb.ServerAddress]*ClusterNode),
|
||||
filerGroups: newClusterNodeGroups(),
|
||||
brokerGroups: newClusterNodeGroups(),
|
||||
}
|
||||
}
|
||||
|
||||
func (cluster *Cluster) getFilers(filerGroup FilerGroup, createIfNotFound bool) *Filers {
|
||||
filers, found := cluster.filerGroup2filers[filerGroup]
|
||||
if !found && createIfNotFound {
|
||||
filers = &Filers{
|
||||
filers: make(map[pb.ServerAddress]*ClusterNode),
|
||||
leaders: &Leaders{},
|
||||
}
|
||||
cluster.filerGroup2filers[filerGroup] = filers
|
||||
}
|
||||
return filers
|
||||
}
|
||||
|
||||
func (cluster *Cluster) AddClusterNode(ns, nodeType string, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
|
||||
filerGroup := FilerGroup(ns)
|
||||
func (cluster *Cluster) getGroupMembers(filerGroup FilerGroupName, nodeType string, createIfNotFound bool) *GroupMembers {
|
||||
switch nodeType {
|
||||
case FilerType:
|
||||
cluster.filersLock.Lock()
|
||||
defer cluster.filersLock.Unlock()
|
||||
filers := cluster.getFilers(filerGroup, true)
|
||||
if existingNode, found := filers.filers[address]; found {
|
||||
existingNode.counter++
|
||||
return nil
|
||||
}
|
||||
filers.filers[address] = &ClusterNode{
|
||||
Address: address,
|
||||
Version: version,
|
||||
counter: 1,
|
||||
CreatedTs: time.Now(),
|
||||
}
|
||||
return cluster.ensureFilerLeaders(filers, true, filerGroup, nodeType, address)
|
||||
return cluster.filerGroups.getGroupMembers(filerGroup, createIfNotFound)
|
||||
case BrokerType:
|
||||
cluster.brokersLock.Lock()
|
||||
defer cluster.brokersLock.Unlock()
|
||||
if existingNode, found := cluster.brokers[address]; found {
|
||||
existingNode.counter++
|
||||
return nil
|
||||
}
|
||||
cluster.brokers[address] = &ClusterNode{
|
||||
Address: address,
|
||||
Version: version,
|
||||
counter: 1,
|
||||
CreatedTs: time.Now(),
|
||||
}
|
||||
return []*master_pb.KeepConnectedResponse{
|
||||
{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
NodeType: nodeType,
|
||||
Address: string(address),
|
||||
IsAdd: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
return cluster.brokerGroups.getGroupMembers(filerGroup, createIfNotFound)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cluster *Cluster) AddClusterNode(ns, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
|
||||
filerGroup := FilerGroupName(ns)
|
||||
switch nodeType {
|
||||
case FilerType:
|
||||
return cluster.filerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
|
||||
case BrokerType:
|
||||
return cluster.brokerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
|
||||
case MasterType:
|
||||
return []*master_pb.KeepConnectedResponse{
|
||||
{
|
||||
@@ -112,44 +180,12 @@ func (cluster *Cluster) AddClusterNode(ns, nodeType string, address pb.ServerAdd
|
||||
}
|
||||
|
||||
func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
|
||||
filerGroup := FilerGroup(ns)
|
||||
filerGroup := FilerGroupName(ns)
|
||||
switch nodeType {
|
||||
case FilerType:
|
||||
cluster.filersLock.Lock()
|
||||
defer cluster.filersLock.Unlock()
|
||||
filers := cluster.getFilers(filerGroup, false)
|
||||
if filers == nil {
|
||||
return nil
|
||||
}
|
||||
if existingNode, found := filers.filers[address]; !found {
|
||||
return nil
|
||||
} else {
|
||||
existingNode.counter--
|
||||
if existingNode.counter <= 0 {
|
||||
delete(filers.filers, address)
|
||||
return cluster.ensureFilerLeaders(filers, false, filerGroup, nodeType, address)
|
||||
}
|
||||
}
|
||||
return cluster.filerGroups.RemoveClusterNode(filerGroup, nodeType, address)
|
||||
case BrokerType:
|
||||
cluster.brokersLock.Lock()
|
||||
defer cluster.brokersLock.Unlock()
|
||||
if existingNode, found := cluster.brokers[address]; !found {
|
||||
return nil
|
||||
} else {
|
||||
existingNode.counter--
|
||||
if existingNode.counter <= 0 {
|
||||
delete(cluster.brokers, address)
|
||||
return []*master_pb.KeepConnectedResponse{
|
||||
{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
NodeType: nodeType,
|
||||
Address: string(address),
|
||||
IsAdd: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
return cluster.brokerGroups.RemoveClusterNode(filerGroup, nodeType, address)
|
||||
case MasterType:
|
||||
return []*master_pb.KeepConnectedResponse{
|
||||
{
|
||||
@@ -164,40 +200,42 @@ func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cluster *Cluster) ListClusterNode(filerGroup FilerGroup, nodeType string) (nodes []*ClusterNode) {
|
||||
func (cluster *Cluster) ListClusterNode(filerGroup FilerGroupName, nodeType string) (nodes []*ClusterNode) {
|
||||
switch nodeType {
|
||||
case FilerType:
|
||||
cluster.filersLock.RLock()
|
||||
defer cluster.filersLock.RUnlock()
|
||||
filers := cluster.getFilers(filerGroup, false)
|
||||
if filers == nil {
|
||||
return
|
||||
}
|
||||
for _, node := range filers.filers {
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
return cluster.filerGroups.ListClusterNode(filerGroup)
|
||||
case BrokerType:
|
||||
cluster.brokersLock.RLock()
|
||||
defer cluster.brokersLock.RUnlock()
|
||||
for _, node := range cluster.brokers {
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
return cluster.brokerGroups.ListClusterNode(filerGroup)
|
||||
case MasterType:
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (cluster *Cluster) IsOneLeader(filerGroup FilerGroup, address pb.ServerAddress) bool {
|
||||
filers := cluster.getFilers(filerGroup, false)
|
||||
if filers == nil {
|
||||
return false
|
||||
func (cluster *Cluster) ListClusterNodeLeaders(filerGroup FilerGroupName, nodeType string) (nodes []pb.ServerAddress) {
|
||||
switch nodeType {
|
||||
case FilerType:
|
||||
return cluster.filerGroups.ListClusterNodeLeaders(filerGroup)
|
||||
case BrokerType:
|
||||
return cluster.brokerGroups.ListClusterNodeLeaders(filerGroup)
|
||||
case MasterType:
|
||||
}
|
||||
return filers.leaders.isOneLeader(address)
|
||||
return
|
||||
}
|
||||
|
||||
func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGroup FilerGroup, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
|
||||
func (cluster *Cluster) IsOneLeader(filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) bool {
|
||||
switch nodeType {
|
||||
case FilerType:
|
||||
return cluster.filerGroups.IsOneLeader(filerGroup, address)
|
||||
case BrokerType:
|
||||
return cluster.brokerGroups.IsOneLeader(filerGroup, address)
|
||||
case MasterType:
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ensureGroupLeaders(m *GroupMembers, isAdd bool, filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
|
||||
if isAdd {
|
||||
if filers.leaders.addLeaderIfVacant(address) {
|
||||
if m.leaders.addLeaderIfVacant(address) {
|
||||
// has added the address as one leader
|
||||
result = append(result, &master_pb.KeepConnectedResponse{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
@@ -220,7 +258,7 @@ func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGrou
|
||||
})
|
||||
}
|
||||
} else {
|
||||
if filers.leaders.removeLeaderIfExists(address) {
|
||||
if m.leaders.removeLeaderIfExists(address) {
|
||||
|
||||
result = append(result, &master_pb.KeepConnectedResponse{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
@@ -236,8 +274,8 @@ func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGrou
|
||||
var shortestDuration int64 = math.MaxInt64
|
||||
now := time.Now()
|
||||
var candidateAddress pb.ServerAddress
|
||||
for _, node := range filers.filers {
|
||||
if filers.leaders.isOneLeader(node.Address) {
|
||||
for _, node := range m.members {
|
||||
if m.leaders.isOneLeader(node.Address) {
|
||||
continue
|
||||
}
|
||||
duration := now.Sub(node.CreatedTs).Nanoseconds()
|
||||
@@ -247,7 +285,7 @@ func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGrou
|
||||
}
|
||||
}
|
||||
if candidateAddress != "" {
|
||||
filers.leaders.addLeaderIfVacant(candidateAddress)
|
||||
m.leaders.addLeaderIfVacant(candidateAddress)
|
||||
// added a new leader
|
||||
result = append(result, &master_pb.KeepConnectedResponse{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"strconv"
|
||||
"sync"
|
||||
@@ -11,29 +11,29 @@ import (
|
||||
func TestClusterAddRemoveNodes(t *testing.T) {
|
||||
c := NewCluster()
|
||||
|
||||
c.AddClusterNode("", "filer", pb.ServerAddress("111:1"), "23.45")
|
||||
c.AddClusterNode("", "filer", pb.ServerAddress("111:2"), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:1"), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:2"), "23.45")
|
||||
assert.Equal(t, []pb.ServerAddress{
|
||||
pb.ServerAddress("111:1"),
|
||||
pb.ServerAddress("111:2"),
|
||||
}, c.getFilers("", false).leaders.GetLeaders())
|
||||
}, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
|
||||
|
||||
c.AddClusterNode("", "filer", pb.ServerAddress("111:3"), "23.45")
|
||||
c.AddClusterNode("", "filer", pb.ServerAddress("111:4"), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:3"), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:4"), "23.45")
|
||||
assert.Equal(t, []pb.ServerAddress{
|
||||
pb.ServerAddress("111:1"),
|
||||
pb.ServerAddress("111:2"),
|
||||
pb.ServerAddress("111:3"),
|
||||
}, c.getFilers("", false).leaders.GetLeaders())
|
||||
}, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
|
||||
|
||||
c.AddClusterNode("", "filer", pb.ServerAddress("111:5"), "23.45")
|
||||
c.AddClusterNode("", "filer", pb.ServerAddress("111:6"), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:5"), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:6"), "23.45")
|
||||
c.RemoveClusterNode("", "filer", pb.ServerAddress("111:4"))
|
||||
assert.Equal(t, []pb.ServerAddress{
|
||||
pb.ServerAddress("111:1"),
|
||||
pb.ServerAddress("111:2"),
|
||||
pb.ServerAddress("111:3"),
|
||||
}, c.getFilers("", false).leaders.GetLeaders())
|
||||
}, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
|
||||
|
||||
// remove oldest
|
||||
c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1"))
|
||||
@@ -41,7 +41,7 @@ func TestClusterAddRemoveNodes(t *testing.T) {
|
||||
pb.ServerAddress("111:6"),
|
||||
pb.ServerAddress("111:2"),
|
||||
pb.ServerAddress("111:3"),
|
||||
}, c.getFilers("", false).leaders.GetLeaders())
|
||||
}, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
|
||||
|
||||
// remove oldest
|
||||
c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1"))
|
||||
@@ -56,7 +56,7 @@ func TestConcurrentAddRemoveNodes(t *testing.T) {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
address := strconv.Itoa(i)
|
||||
c.AddClusterNode("", "filer", pb.ServerAddress(address), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress(address), "23.45")
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func ListExistingPeerUpdates(master pb.ServerAddress, grpcDialOption grpc.DialOption, filerGroup string, clientType string) (existingNodes []*master_pb.ClusterNodeUpdate) {
|
||||
|
||||
if grpcErr := pb.WithMasterClient(false, master, grpcDialOption, false, func(client master_pb.SeaweedClient) error {
|
||||
resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
|
||||
ClientType: clientType,
|
||||
FilerGroup: filerGroup,
|
||||
})
|
||||
|
||||
glog.V(0).Infof("the cluster has %d %s\n", len(resp.ClusterNodes), clientType)
|
||||
for _, node := range resp.ClusterNodes {
|
||||
existingNodes = append(existingNodes, &master_pb.ClusterNodeUpdate{
|
||||
NodeType: FilerType,
|
||||
Address: node.Address,
|
||||
IsLeader: node.IsLeader,
|
||||
IsAdd: true,
|
||||
CreatedAtNs: node.CreatedAtNs,
|
||||
})
|
||||
}
|
||||
return err
|
||||
}); grpcErr != nil {
|
||||
glog.V(0).Infof("connect to %s: %v", master, grpcErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -2,9 +2,9 @@ package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
|
||||
"github.com/posener/complete"
|
||||
completeinstall "github.com/posener/complete/cmd/install"
|
||||
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
|
||||
@@ -2,15 +2,15 @@ package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -3,7 +3,7 @@ package command
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"io"
|
||||
"math"
|
||||
"math/rand"
|
||||
@@ -16,11 +16,11 @@ import (
|
||||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
)
|
||||
|
||||
type BenchmarkOptions struct {
|
||||
@@ -129,7 +129,7 @@ func runBenchmark(cmd *Command, args []string) bool {
|
||||
defer pprof.StopCPUProfile()
|
||||
}
|
||||
|
||||
b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "", "client", "", "", pb.ServerAddresses(*b.masters).ToAddressMap())
|
||||
b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "", "client", "", "", "", pb.ServerAddresses(*b.masters).ToAddressMap())
|
||||
go b.masterClient.KeepConnectedToMaster()
|
||||
b.masterClient.WaitUntilConnected()
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
|
||||
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
|
||||
)
|
||||
|
||||
var Commands = []*Command{
|
||||
@@ -28,12 +28,12 @@ var Commands = []*Command{
|
||||
cmdFilerSynchronize,
|
||||
cmdFix,
|
||||
cmdFuse,
|
||||
cmdIam,
|
||||
cmdMaster,
|
||||
cmdMasterFollower,
|
||||
cmdMount,
|
||||
cmdMqBroker,
|
||||
cmdS3,
|
||||
cmdIam,
|
||||
cmdMsgBroker,
|
||||
cmdScaffold,
|
||||
cmdServer,
|
||||
cmdShell,
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -10,10 +10,10 @@ import (
|
||||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -13,13 +13,13 @@ import (
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+34
-26
@@ -12,14 +12,14 @@ import (
|
||||
|
||||
"google.golang.org/grpc/reflection"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
weed_server "github.com/chrislusf/seaweedfs/weed/server"
|
||||
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
|
||||
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -59,6 +59,7 @@ type FilerOptions struct {
|
||||
debugPort *int
|
||||
localSocket *string
|
||||
showUIDirectoryDelete *bool
|
||||
downloadMaxMBps *int
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -87,12 +88,14 @@ func init() {
|
||||
f.debugPort = cmdFiler.Flag.Int("debug.port", 6060, "http port for debugging")
|
||||
f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
|
||||
f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button")
|
||||
f.downloadMaxMBps = cmdFiler.Flag.Int("downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
|
||||
|
||||
// start s3 on filer
|
||||
filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
|
||||
filerS3Options.port = cmdFiler.Flag.Int("s3.port", 8333, "s3 server http listen port")
|
||||
filerS3Options.portGrpc = cmdFiler.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port")
|
||||
filerS3Options.domainName = cmdFiler.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
|
||||
filerS3Options.dataCenter = cmdFiler.Flag.String("s3.dataCenter", "", "prefer to read and write to volumes in this data center")
|
||||
filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file")
|
||||
filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
|
||||
filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file")
|
||||
@@ -167,29 +170,32 @@ func runFiler(cmd *Command, args []string) bool {
|
||||
filerS3Options.filer = &filerAddress
|
||||
filerS3Options.bindIp = f.bindIp
|
||||
filerS3Options.localFilerSocket = f.localSocket
|
||||
go func() {
|
||||
time.Sleep(startDelay * time.Second)
|
||||
if *f.dataCenter != "" && *filerS3Options.dataCenter == "" {
|
||||
filerS3Options.dataCenter = f.dataCenter
|
||||
}
|
||||
go func(delay time.Duration) {
|
||||
time.Sleep(delay * time.Second)
|
||||
filerS3Options.startS3Server()
|
||||
}()
|
||||
}(startDelay)
|
||||
startDelay++
|
||||
}
|
||||
|
||||
if *filerStartWebDav {
|
||||
filerWebDavOptions.filer = &filerAddress
|
||||
go func() {
|
||||
time.Sleep(startDelay * time.Second)
|
||||
go func(delay time.Duration) {
|
||||
time.Sleep(delay * time.Second)
|
||||
filerWebDavOptions.startWebDav()
|
||||
}()
|
||||
}(startDelay)
|
||||
startDelay++
|
||||
}
|
||||
|
||||
if *filerStartIam {
|
||||
filerIamOptions.filer = &filerAddress
|
||||
filerIamOptions.masters = f.mastersString
|
||||
go func() {
|
||||
time.Sleep(startDelay * time.Second)
|
||||
go func(delay time.Duration) {
|
||||
time.Sleep(delay * time.Second)
|
||||
filerIamOptions.startIamServer()
|
||||
}()
|
||||
}(startDelay)
|
||||
}
|
||||
|
||||
f.masters = pb.ServerAddresses(*f.mastersString).ToAddressMap()
|
||||
@@ -235,6 +241,7 @@ func (fo *FilerOptions) startFiler() {
|
||||
SaveToFilerLimit: int64(*fo.saveToFilerLimit),
|
||||
ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
|
||||
ShowUIDirectoryDelete: *fo.showUIDirectoryDelete,
|
||||
DownloadMaxBytesPs: int64(*fo.downloadMaxMBps) * 1024 * 1024,
|
||||
})
|
||||
if nfs_err != nil {
|
||||
glog.Fatalf("Filer startup error: %v", nfs_err)
|
||||
@@ -243,7 +250,7 @@ func (fo *FilerOptions) startFiler() {
|
||||
if *fo.publicPort != 0 {
|
||||
publicListeningAddress := util.JoinHostPort(*fo.bindIp, *fo.publicPort)
|
||||
glog.V(0).Infoln("Start Seaweed filer server", util.Version(), "public at", publicListeningAddress)
|
||||
publicListener, localPublicListner, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0)
|
||||
publicListener, localPublicListener, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0)
|
||||
if e != nil {
|
||||
glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
|
||||
}
|
||||
@@ -252,9 +259,9 @@ func (fo *FilerOptions) startFiler() {
|
||||
glog.Fatalf("Volume server fail to serve public: %v", e)
|
||||
}
|
||||
}()
|
||||
if localPublicListner != nil {
|
||||
if localPublicListener != nil {
|
||||
go func() {
|
||||
if e := http.Serve(localPublicListner, publicVolumeMux); e != nil {
|
||||
if e := http.Serve(localPublicListener, publicVolumeMux); e != nil {
|
||||
glog.Errorf("Volume server fail to serve public: %v", e)
|
||||
}
|
||||
}()
|
||||
@@ -286,17 +293,18 @@ func (fo *FilerOptions) startFiler() {
|
||||
|
||||
httpS := &http.Server{Handler: defaultMux}
|
||||
if runtime.GOOS != "windows" {
|
||||
if *fo.localSocket == "" {
|
||||
*fo.localSocket = fmt.Sprintf("/tmp/seaweefs-filer-%d.sock", *fo.port)
|
||||
localSocket := *fo.localSocket
|
||||
if localSocket == "" {
|
||||
localSocket = fmt.Sprintf("/tmp/seaweedfs-filer-%d.sock", *fo.port)
|
||||
}
|
||||
if err := os.Remove(*fo.localSocket); err != nil && !os.IsNotExist(err) {
|
||||
glog.Fatalf("Failed to remove %s, error: %s", *fo.localSocket, err.Error())
|
||||
if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) {
|
||||
glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
|
||||
}
|
||||
go func() {
|
||||
// start on local unix socket
|
||||
filerSocketListener, err := net.Listen("unix", *fo.localSocket)
|
||||
filerSocketListener, err := net.Listen("unix", localSocket)
|
||||
if err != nil {
|
||||
glog.Fatalf("Failed to listen on %s: %v", *fo.localSocket, err)
|
||||
glog.Fatalf("Failed to listen on %s: %v", localSocket, err)
|
||||
}
|
||||
httpS.Serve(filerSocketListener)
|
||||
}()
|
||||
|
||||
@@ -2,11 +2,11 @@ package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/replication/source"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/source"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
"time"
|
||||
)
|
||||
@@ -15,6 +15,7 @@ type FilerBackupOptions struct {
|
||||
isActivePassive *bool
|
||||
filer *string
|
||||
path *string
|
||||
excludePaths *string
|
||||
debug *bool
|
||||
proxyByFiler *bool
|
||||
timeAgo *time.Duration
|
||||
@@ -28,6 +29,7 @@ func init() {
|
||||
cmdFilerBackup.Run = runFilerBackup // break init cycle
|
||||
filerBackupOptions.filer = cmdFilerBackup.Flag.String("filer", "localhost:8888", "filer of one SeaweedFS cluster")
|
||||
filerBackupOptions.path = cmdFilerBackup.Flag.String("filerPath", "/", "directory to sync on filer")
|
||||
filerBackupOptions.excludePaths = cmdFilerBackup.Flag.String("filerExcludePaths", "", "exclude directories to sync on filer")
|
||||
filerBackupOptions.proxyByFiler = cmdFilerBackup.Flag.Bool("filerProxy", false, "read and write file chunks by filer instead of volume servers")
|
||||
filerBackupOptions.debug = cmdFilerBackup.Flag.Bool("debug", false, "debug mode to print out received files")
|
||||
filerBackupOptions.timeAgo = cmdFilerBackup.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
|
||||
@@ -55,9 +57,11 @@ func runFilerBackup(cmd *Command, args []string) bool {
|
||||
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
|
||||
|
||||
clientId := util.RandomInt32()
|
||||
var clientEpoch int32
|
||||
|
||||
for {
|
||||
err := doFilerBackup(grpcDialOption, &filerBackupOptions, clientId)
|
||||
clientEpoch++
|
||||
err := doFilerBackup(grpcDialOption, &filerBackupOptions, clientId, clientEpoch)
|
||||
if err != nil {
|
||||
glog.Errorf("backup from %s: %v", *filerBackupOptions.filer, err)
|
||||
time.Sleep(1747 * time.Millisecond)
|
||||
@@ -71,7 +75,7 @@ const (
|
||||
BackupKeyPrefix = "backup."
|
||||
)
|
||||
|
||||
func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions, clientId int32) error {
|
||||
func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions, clientId int32, clientEpoch int32) error {
|
||||
|
||||
// find data sink
|
||||
config := util.GetViper()
|
||||
@@ -82,6 +86,7 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
|
||||
|
||||
sourceFiler := pb.ServerAddress(*backupOption.filer)
|
||||
sourcePath := *backupOption.path
|
||||
excludePaths := util.StringSplit(*backupOption.excludePaths, ",")
|
||||
timeAgo := *backupOption.timeAgo
|
||||
targetPath := dataSink.GetSinkToDirectory()
|
||||
debug := *backupOption.debug
|
||||
@@ -104,16 +109,20 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
|
||||
|
||||
// create filer sink
|
||||
filerSource := &source.FilerSource{}
|
||||
filerSource.DoInitialize(sourceFiler.ToHttpAddress(), sourceFiler.ToGrpcAddress(), sourcePath, *backupOption.proxyByFiler)
|
||||
filerSource.DoInitialize(
|
||||
sourceFiler.ToHttpAddress(),
|
||||
sourceFiler.ToGrpcAddress(),
|
||||
sourcePath,
|
||||
*backupOption.proxyByFiler)
|
||||
dataSink.SetSourceFiler(filerSource)
|
||||
|
||||
processEventFn := genProcessFunction(sourcePath, targetPath, dataSink, debug)
|
||||
processEventFn := genProcessFunction(sourcePath, targetPath, excludePaths, dataSink, debug)
|
||||
|
||||
processEventFnWithOffset := pb.AddOffsetFunc(processEventFn, 3*time.Second, func(counter int64, lastTsNs int64) error {
|
||||
glog.V(0).Infof("backup %s progressed to %v %0.2f/sec", sourceFiler, time.Unix(0, lastTsNs), float64(counter)/float64(3))
|
||||
return setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId), lastTsNs)
|
||||
})
|
||||
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(), clientId, sourcePath, nil, startFrom.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(), clientId, clientEpoch, sourcePath, nil, startFrom.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
|
||||
|
||||
}
|
||||
@@ -3,17 +3,17 @@ package command
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"google.golang.org/grpc"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
+123
-188
@@ -14,15 +14,15 @@ import (
|
||||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/chrislusf/seaweedfs/weed/util/grace"
|
||||
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -31,21 +31,21 @@ var (
|
||||
)
|
||||
|
||||
type CopyOptions struct {
|
||||
include *string
|
||||
replication *string
|
||||
collection *string
|
||||
ttl *string
|
||||
diskType *string
|
||||
maxMB *int
|
||||
masterClient *wdclient.MasterClient
|
||||
concurrenctFiles *int
|
||||
concurrenctChunks *int
|
||||
grpcDialOption grpc.DialOption
|
||||
masters []string
|
||||
cipher bool
|
||||
ttlSec int32
|
||||
checkSize *bool
|
||||
verbose *bool
|
||||
include *string
|
||||
replication *string
|
||||
collection *string
|
||||
ttl *string
|
||||
diskType *string
|
||||
maxMB *int
|
||||
masterClient *wdclient.MasterClient
|
||||
concurrentFiles *int
|
||||
concurrentChunks *int
|
||||
grpcDialOption grpc.DialOption
|
||||
masters []string
|
||||
cipher bool
|
||||
ttlSec int32
|
||||
checkSize *bool
|
||||
verbose *bool
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -57,8 +57,8 @@ func init() {
|
||||
copy.ttl = cmdFilerCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
|
||||
copy.diskType = cmdFilerCopy.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
|
||||
copy.maxMB = cmdFilerCopy.Flag.Int("maxMB", 4, "split files larger than the limit")
|
||||
copy.concurrenctFiles = cmdFilerCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
|
||||
copy.concurrenctChunks = cmdFilerCopy.Flag.Int("concurrentChunks", 8, "concurrent chunk copy goroutines for each file")
|
||||
copy.concurrentFiles = cmdFilerCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
|
||||
copy.concurrentChunks = cmdFilerCopy.Flag.Int("concurrentChunks", 8, "concurrent chunk copy goroutines for each file")
|
||||
copy.checkSize = cmdFilerCopy.Flag.Bool("check.size", false, "copy when the target file size is different from the source file")
|
||||
copy.verbose = cmdFilerCopy.Flag.Bool("verbose", false, "print out details during copying")
|
||||
}
|
||||
@@ -141,7 +141,7 @@ func runCopy(cmd *Command, args []string) bool {
|
||||
grace.SetupProfiling("filer.copy.cpu.pprof", "filer.copy.mem.pprof")
|
||||
}
|
||||
|
||||
fileCopyTaskChan := make(chan FileCopyTask, *copy.concurrenctFiles)
|
||||
fileCopyTaskChan := make(chan FileCopyTask, *copy.concurrentFiles)
|
||||
|
||||
go func() {
|
||||
defer close(fileCopyTaskChan)
|
||||
@@ -152,7 +152,7 @@ func runCopy(cmd *Command, args []string) bool {
|
||||
}
|
||||
}
|
||||
}()
|
||||
for i := 0; i < *copy.concurrenctFiles; i++ {
|
||||
for i := 0; i < *copy.concurrentFiles; i++ {
|
||||
waitGroup.Add(1)
|
||||
go func() {
|
||||
defer waitGroup.Done()
|
||||
@@ -331,8 +331,6 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
|
||||
var mimeType string
|
||||
|
||||
var chunks []*filer_pb.FileChunk
|
||||
var assignResult *filer_pb.AssignVolumeResponse
|
||||
var assignError error
|
||||
|
||||
if task.fileMode&os.ModeDir == 0 && task.fileSize > 0 {
|
||||
|
||||
@@ -342,66 +340,32 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
|
||||
return err
|
||||
}
|
||||
|
||||
err = util.Retry("upload", func() error {
|
||||
// assign a volume
|
||||
assignErr := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
||||
request := &filer_pb.AssignVolumeRequest{
|
||||
Count: 1,
|
||||
Replication: *worker.options.replication,
|
||||
Collection: *worker.options.collection,
|
||||
TtlSec: worker.options.ttlSec,
|
||||
DiskType: *worker.options.diskType,
|
||||
Path: task.destinationUrlPath,
|
||||
}
|
||||
|
||||
assignResult, assignError = client.AssignVolume(context.Background(), request)
|
||||
if assignError != nil {
|
||||
return fmt.Errorf("assign volume failure %v: %v", request, assignError)
|
||||
}
|
||||
if assignResult.Error != "" {
|
||||
return fmt.Errorf("assign volume failure %v: %v", request, assignResult.Error)
|
||||
}
|
||||
if assignResult.Location.Url == "" {
|
||||
return fmt.Errorf("assign volume failure %v: %v", request, assignResult)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if assignErr != nil {
|
||||
return assignErr
|
||||
}
|
||||
|
||||
// upload data
|
||||
targetUrl := "http://" + assignResult.Location.Url + "/" + assignResult.FileId
|
||||
uploadOption := &operation.UploadOption{
|
||||
UploadUrl: targetUrl,
|
||||
finalFileId, uploadResult, flushErr, _ := operation.UploadWithRetry(
|
||||
worker,
|
||||
&filer_pb.AssignVolumeRequest{
|
||||
Count: 1,
|
||||
Replication: *worker.options.replication,
|
||||
Collection: *worker.options.collection,
|
||||
TtlSec: worker.options.ttlSec,
|
||||
DiskType: *worker.options.diskType,
|
||||
Path: task.destinationUrlPath,
|
||||
},
|
||||
&operation.UploadOption{
|
||||
Filename: fileName,
|
||||
Cipher: worker.options.cipher,
|
||||
IsInputCompressed: false,
|
||||
MimeType: mimeType,
|
||||
PairMap: nil,
|
||||
Jwt: security.EncodedJwt(assignResult.Auth),
|
||||
}
|
||||
uploadResult, err := operation.UploadData(data, uploadOption)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
|
||||
}
|
||||
if uploadResult.Error != "" {
|
||||
return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
|
||||
}
|
||||
if *worker.options.verbose {
|
||||
fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)
|
||||
}
|
||||
|
||||
fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName)
|
||||
chunks = append(chunks, uploadResult.ToPbFileChunk(assignResult.FileId, 0))
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload %v: %v\n", fileName, err)
|
||||
},
|
||||
func(host, fileId string) string {
|
||||
return fmt.Sprintf("http://%s/%s", host, fileId)
|
||||
},
|
||||
util.NewBytesReader(data),
|
||||
)
|
||||
if flushErr != nil {
|
||||
return flushErr
|
||||
}
|
||||
|
||||
chunks = append(chunks, uploadResult.ToPbFileChunk(finalFileId, 0))
|
||||
}
|
||||
|
||||
if err := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
@@ -441,10 +405,9 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
|
||||
|
||||
chunksChan := make(chan *filer_pb.FileChunk, chunkCount)
|
||||
|
||||
concurrentChunks := make(chan struct{}, *worker.options.concurrenctChunks)
|
||||
concurrentChunks := make(chan struct{}, *worker.options.concurrentChunks)
|
||||
var wg sync.WaitGroup
|
||||
var uploadError error
|
||||
var collection, replication string
|
||||
|
||||
fmt.Printf("uploading %s in %d chunks ...\n", fileName, chunkCount)
|
||||
for i := int64(0); i < int64(chunkCount) && uploadError == nil; i++ {
|
||||
@@ -455,64 +418,41 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
|
||||
wg.Done()
|
||||
<-concurrentChunks
|
||||
}()
|
||||
// assign a volume
|
||||
var assignResult *filer_pb.AssignVolumeResponse
|
||||
var assignError error
|
||||
err := util.Retry("assignVolume", func() error {
|
||||
return pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
request := &filer_pb.AssignVolumeRequest{
|
||||
Count: 1,
|
||||
Replication: *worker.options.replication,
|
||||
Collection: *worker.options.collection,
|
||||
TtlSec: worker.options.ttlSec,
|
||||
DiskType: *worker.options.diskType,
|
||||
Path: task.destinationUrlPath + fileName,
|
||||
}
|
||||
|
||||
assignResult, assignError = client.AssignVolume(context.Background(), request)
|
||||
if assignError != nil {
|
||||
return fmt.Errorf("assign volume failure %v: %v", request, assignError)
|
||||
}
|
||||
if assignResult.Error != "" {
|
||||
return fmt.Errorf("assign volume failure %v: %v", request, assignResult.Error)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
})
|
||||
fileId, uploadResult, err, _ := operation.UploadWithRetry(
|
||||
worker,
|
||||
&filer_pb.AssignVolumeRequest{
|
||||
Count: 1,
|
||||
Replication: *worker.options.replication,
|
||||
Collection: *worker.options.collection,
|
||||
TtlSec: worker.options.ttlSec,
|
||||
DiskType: *worker.options.diskType,
|
||||
Path: task.destinationUrlPath + fileName,
|
||||
},
|
||||
&operation.UploadOption{
|
||||
Filename: fileName + "-" + strconv.FormatInt(i+1, 10),
|
||||
Cipher: worker.options.cipher,
|
||||
IsInputCompressed: false,
|
||||
MimeType: "",
|
||||
PairMap: nil,
|
||||
},
|
||||
func(host, fileId string) string {
|
||||
return fmt.Sprintf("http://%s/%s", host, fileId)
|
||||
},
|
||||
io.NewSectionReader(f, i*chunkSize, chunkSize),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
uploadError = fmt.Errorf("Failed to assign from %v: %v\n", worker.options.masters, err)
|
||||
return
|
||||
}
|
||||
|
||||
targetUrl := "http://" + assignResult.Location.Url + "/" + assignResult.FileId
|
||||
if collection == "" {
|
||||
collection = assignResult.Collection
|
||||
}
|
||||
if replication == "" {
|
||||
replication = assignResult.Replication
|
||||
}
|
||||
|
||||
uploadOption := &operation.UploadOption{
|
||||
UploadUrl: targetUrl,
|
||||
Filename: fileName + "-" + strconv.FormatInt(i+1, 10),
|
||||
Cipher: worker.options.cipher,
|
||||
IsInputCompressed: false,
|
||||
MimeType: "",
|
||||
PairMap: nil,
|
||||
Jwt: security.EncodedJwt(assignResult.Auth),
|
||||
}
|
||||
uploadResult, err, _ := operation.Upload(io.NewSectionReader(f, i*chunkSize, chunkSize), uploadOption)
|
||||
if err != nil {
|
||||
uploadError = fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
|
||||
uploadError = fmt.Errorf("upload data %v: %v\n", fileName, err)
|
||||
return
|
||||
}
|
||||
if uploadResult.Error != "" {
|
||||
uploadError = fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
|
||||
uploadError = fmt.Errorf("upload %v result: %v\n", fileName, uploadResult.Error)
|
||||
return
|
||||
}
|
||||
chunksChan <- uploadResult.ToPbFileChunk(assignResult.FileId, i*chunkSize)
|
||||
chunksChan <- uploadResult.ToPbFileChunk(fileId, i*chunkSize)
|
||||
|
||||
fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
|
||||
fmt.Printf("uploaded %s-%d [%d,%d)\n", fileName, i+1, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
@@ -590,62 +530,57 @@ func detectMimeType(f *os.File) string {
|
||||
return mimeType
|
||||
}
|
||||
|
||||
func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, offset int64) (chunk *filer_pb.FileChunk, collection, replication string, err error) {
|
||||
func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, offset int64) (chunk *filer_pb.FileChunk, err error) {
|
||||
|
||||
var fileId, host string
|
||||
var auth security.EncodedJwt
|
||||
finalFileId, uploadResult, flushErr, _ := operation.UploadWithRetry(
|
||||
worker,
|
||||
&filer_pb.AssignVolumeRequest{
|
||||
Count: 1,
|
||||
Replication: *worker.options.replication,
|
||||
Collection: *worker.options.collection,
|
||||
TtlSec: worker.options.ttlSec,
|
||||
DiskType: *worker.options.diskType,
|
||||
Path: name,
|
||||
},
|
||||
&operation.UploadOption{
|
||||
Filename: name,
|
||||
Cipher: worker.options.cipher,
|
||||
IsInputCompressed: false,
|
||||
MimeType: "",
|
||||
PairMap: nil,
|
||||
},
|
||||
func(host, fileId string) string {
|
||||
return fmt.Sprintf("http://%s/%s", host, fileId)
|
||||
},
|
||||
reader,
|
||||
)
|
||||
|
||||
if flushErr := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
assignErr := util.Retry("assignVolume", func() error {
|
||||
request := &filer_pb.AssignVolumeRequest{
|
||||
Count: 1,
|
||||
Replication: *worker.options.replication,
|
||||
Collection: *worker.options.collection,
|
||||
TtlSec: worker.options.ttlSec,
|
||||
DiskType: *worker.options.diskType,
|
||||
Path: name,
|
||||
}
|
||||
|
||||
resp, err := client.AssignVolume(ctx, request)
|
||||
if err != nil {
|
||||
return fmt.Errorf("assign volume failure %v: %v", request, err)
|
||||
}
|
||||
if resp.Error != "" {
|
||||
return fmt.Errorf("assign volume failure %v: %v", request, resp.Error)
|
||||
}
|
||||
|
||||
fileId, host, auth = resp.FileId, resp.Location.Url, security.EncodedJwt(resp.Auth)
|
||||
collection, replication = resp.Collection, resp.Replication
|
||||
|
||||
return nil
|
||||
})
|
||||
if assignErr != nil {
|
||||
return assignErr
|
||||
}
|
||||
|
||||
return nil
|
||||
}); flushErr != nil {
|
||||
return nil, collection, replication, fmt.Errorf("filerGrpcAddress assign volume: %v", flushErr)
|
||||
}
|
||||
|
||||
uploadOption := &operation.UploadOption{
|
||||
UploadUrl: fmt.Sprintf("http://%s/%s", host, fileId),
|
||||
Filename: name,
|
||||
Cipher: worker.options.cipher,
|
||||
IsInputCompressed: false,
|
||||
MimeType: "",
|
||||
PairMap: nil,
|
||||
Jwt: auth,
|
||||
}
|
||||
uploadResult, flushErr, _ := operation.Upload(reader, uploadOption)
|
||||
if flushErr != nil {
|
||||
return nil, collection, replication, fmt.Errorf("upload data: %v", flushErr)
|
||||
return nil, fmt.Errorf("upload data: %v", flushErr)
|
||||
}
|
||||
if uploadResult.Error != "" {
|
||||
return nil, collection, replication, fmt.Errorf("upload result: %v", uploadResult.Error)
|
||||
return nil, fmt.Errorf("upload result: %v", uploadResult.Error)
|
||||
}
|
||||
return uploadResult.ToPbFileChunk(fileId, offset), collection, replication, nil
|
||||
return uploadResult.ToPbFileChunk(finalFileId, offset), nil
|
||||
}
|
||||
|
||||
var _ = filer_pb.FilerClient(&FileCopyWorker{})
|
||||
|
||||
func (worker *FileCopyWorker) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) (err error) {
|
||||
|
||||
filerGrpcAddress := worker.filerAddress.ToGrpcAddress()
|
||||
err = pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
|
||||
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
|
||||
return fn(client)
|
||||
}, filerGrpcAddress, false, worker.options.grpcDialOption)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (worker *FileCopyWorker) AdjustedUrl(location *filer_pb.Location) string {
|
||||
return location.Url
|
||||
}
|
||||
|
||||
func (worker *FileCopyWorker) GetDataCenter() string {
|
||||
return ""
|
||||
}
|
||||
@@ -3,17 +3,17 @@ package command
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/spf13/viper"
|
||||
"google.golang.org/grpc"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -27,8 +27,9 @@ type FilerMetaBackupOptions struct {
|
||||
restart *bool
|
||||
backupFilerConfig *string
|
||||
|
||||
store filer.FilerStore
|
||||
clientId int32
|
||||
store filer.FilerStore
|
||||
clientId int32
|
||||
clientEpoch int32
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -194,7 +195,8 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
|
||||
return metaBackup.setOffset(lastTime)
|
||||
})
|
||||
|
||||
return pb.FollowMetadata(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, "meta_backup", metaBackup.clientId,
|
||||
metaBackup.clientEpoch++
|
||||
return pb.FollowMetadata(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, "meta_backup", metaBackup.clientId, metaBackup.clientEpoch,
|
||||
*metaBackup.filerDirectory, nil, startTime.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
|
||||
|
||||
}
|
||||
@@ -232,3 +234,7 @@ func (metaBackup *FilerMetaBackupOptions) WithFilerClient(streamingMode bool, fn
|
||||
func (metaBackup *FilerMetaBackupOptions) AdjustedUrl(location *filer_pb.Location) string {
|
||||
return location.Url
|
||||
}
|
||||
|
||||
func (metaBackup *FilerMetaBackupOptions) GetDataCenter() string {
|
||||
return ""
|
||||
}
|
||||
@@ -2,16 +2,16 @@ package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/golang/protobuf/jsonpb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -88,11 +88,8 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
jsonpbMarshaler := jsonpb.Marshaler{
|
||||
EmitDefaults: false,
|
||||
}
|
||||
eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
jsonpbMarshaler.Marshal(os.Stdout, resp)
|
||||
filer.ProtoToText(os.Stdout, resp)
|
||||
fmt.Fprintln(os.Stdout)
|
||||
return nil
|
||||
}
|
||||
@@ -110,7 +107,7 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
|
||||
untilTsNs = time.Now().Add(-*tailStop).UnixNano()
|
||||
}
|
||||
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpcDialOption, "tail", clientId, *tailTarget, nil,
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpcDialOption, "tail", clientId, 0, *tailTarget, nil,
|
||||
time.Now().Add(-*tailStart).UnixNano(), untilTsNs, 0, func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
if !shouldPrint(resp) {
|
||||
return nil
|
||||
|
||||
@@ -5,10 +5,10 @@ package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
elastic "github.com/olivere/elastic/v7"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"strings"
|
||||
)
|
||||
|
||||
|
||||
Loaded 100 of 755 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user