-
Notifications
You must be signed in to change notification settings - Fork 4
/
assignReplicaConstraints.sh
executable file
·215 lines (196 loc) · 9.27 KB
/
assignReplicaConstraints.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
#!/bin/bash
show_usage () {
echo "Usage: $0 [options [parameters]]"
echo ""
echo "Options:"
echo " --help -- Prints this page"
echo " --dry-run -- Allows you to test all lthe configs and confirm that your command file bootstrap servers are working"
echo " --bootstrap-server [bootstrap-server-list]"
echo " --command-config [command-config-file]"
echo " --replica-placement-filepath [replica-placement-filepath]"
echo " --describe-all-topics-only -- Use this switch to skip all the other activities and only describe all the topics in the cluster for verification."
return 0
}
if [[ $# -eq 0 ]];then
echo "No input arguments provided."
show_usage
exit 1
fi
echo "============================================================================"
DRY_RUN_ENABLED=false
ONLY_DESCRIBE_TOPICS=false
while [ ! -z "$1" ]
do
if [[ "$1" == "--help" ]]
then
show_usage
exit 0
elif [[ "$1" == "--bootstrap-server" ]]
then
if [[ "$2" == --* ]] || [[ -z "$2" ]]
then
echo "No Value provided for "$1". Please ensure proper values are provided"
show_usage
exit 1
fi
SCRIPT_BOOTSTRAP_SERVERS="$2"
echo "Bootstrap Servers are: ${SCRIPT_BOOTSTRAP_SERVERS}"
shift
elif [[ "$1" == "--command-config" ]]
then
if [[ "$2" == --* ]] || [[ -z "$2" ]]
then
echo "No Value provided for "$1". Please ensure proper values are provided"
show_usage
exit 1
fi
SCRIPT_COMMAND_CONFIG="$2"
echo "Command Config File path is: ${SCRIPT_COMMAND_CONFIG}"
shift
elif [[ "$1" == "--replica-placement-filepath" ]]
then
if [[ "$2" == --* ]] || [[ -z "$2" ]]
then
echo "No Value provided for "$1". Please ensure proper values are provided"
show_usage
exit 1
fi
SCRIPT_REPLICA_PLACEMENT_FILE_NAME="$2"
echo "Replica Placement file is: ${SCRIPT_REPLICA_PLACEMENT_FILE_NAME}"
shift
elif [[ "$1" == "--dry-run" ]]
then
DRY_RUN_ENABLED=true
echo "Dry run is currently enabled"
elif [[ "$1" == "--describe-all-topics-only" ]]
then
ONLY_DESCRIBE_TOPICS=true
echo "Only the topics will be described. If Dry run is enabled, this step will not be executed."
fi
shift
done
SCRIPT_TOPIC_FILE_NAME=(${PWD}/`date +"%s"`_alltopics.txt)
EXECUTION_TOPIC_FILE_NAME=(${PWD}/`date +"%s"`_filterlist.txt)
if [[ -z "$SCRIPT_BOOTSTRAP_SERVERS" ]] || [[ -z "$SCRIPT_REPLICA_PLACEMENT_FILE_NAME" ]]
then
echo "--bootstrap-server and --replica-placement-filepath are required for execution."
show_usage
exit 1
fi
if [[ "$DRY_RUN_ENABLED" = true ]]
then
echo "Fetching the list of topics to ensure that all settings are working properly"
echo "============================================================================"
if [[ -z "${SCRIPT_COMMAND_CONFIG}" ]]
then
echo "Executing without Command Config"
kafka-topics --bootstrap-server ${SCRIPT_BOOTSTRAP_SERVERS} --list
else
echo "Executing with Command Config"
kafka-topics --bootstrap-server ${SCRIPT_BOOTSTRAP_SERVERS} --command-config ${SCRIPT_COMMAND_CONFIG} --list
fi
echo "============================================================================"
echo "Ensure that all the topics are listed in the above output. If some topics are missing, the current user provided in the command config does not have full permissions."
echo "If Confluent RBAC is enabled, ensure that the user is SystemAdmin to allow all necessary operations."
echo "============================================================================"
echo "This script does not validate the Replica placement file for correctness. Please ensure that the file is correct before removing the druy run switch."
echo "Please find below the contents of the Replica Placement file: "
cat ${SCRIPT_REPLICA_PLACEMENT_FILE_NAME}
echo "============================================================================"
exit 0
fi
echo "============================================================================"
echo "Getting the list of topics"
if [[ -z "${SCRIPT_COMMAND_CONFIG}" ]]
then
echo "Executing without Command Config"
kafka-topics --bootstrap-server ${SCRIPT_BOOTSTRAP_SERVERS} --list > ${SCRIPT_TOPIC_FILE_NAME}
else
echo "Executing with Command Config"
kafka-topics --bootstrap-server ${SCRIPT_BOOTSTRAP_SERVERS} --command-config ${SCRIPT_COMMAND_CONFIG} --list > ${SCRIPT_TOPIC_FILE_NAME}
fi
if [[ "$ONLY_DESCRIBE_TOPICS" == true ]]
then
echo "============================================================================"
echo "Describing all topics"
cat ${SCRIPT_TOPIC_FILE_NAME} | while read topic_name
do
if [[ -z "${SCRIPT_COMMAND_CONFIG}" ]]
then
echo "Executing without Command Config"
kafka-topics --bootstrap-server ${SCRIPT_BOOTSTRAP_SERVERS} --topic ${topic_name} --describe
else
echo "Executing with Command Config"
kafka-topics --bootstrap-server ${SCRIPT_BOOTSTRAP_SERVERS} --command-config ${SCRIPT_COMMAND_CONFIG} --topic ${topic_name} --describe
fi
done
rm ${SCRIPT_TOPIC_FILE_NAME}
echo "============================================================================"
exit 0
fi
echo "============================================================================"
echo "The output of this step is available in ${SCRIPT_TOPIC_FILE_NAME}"
echo "If there are multiple unwanted lines due to some issues in java execution, remove them from the file before proceedding to the next step."
echo "============================================================================"
read -p "Press enter to continue once you have validated the file and are happy to proceed further."
echo "============================================================================"
echo "============================================================================"
cat ${SCRIPT_TOPIC_FILE_NAME} | while read topic_name
do
echo "======================"
echo "Checking ${topic_name}"
if [[ -z "${SCRIPT_COMMAND_CONFIG}" ]]
then
OUTPUT_TOPIC_NAME=$(kafka-topics --bootstrap-server ${SCRIPT_BOOTSTRAP_SERVERS} --topic ${topic_name} --describe | grep "Configs:" | grep -v "confluent.placement.constraints={" | awk '{print $2}')
else
OUTPUT_TOPIC_NAME=$(kafka-topics --bootstrap-server ${SCRIPT_BOOTSTRAP_SERVERS} --command-config ${SCRIPT_COMMAND_CONFIG} --topic ${topic_name} --describe | grep "Configs:" | grep -v "confluent.placement.constraints={" | awk '{print $2}')
fi
if [[ -z "$OUTPUT_TOPIC_NAME" ]]
then
echo "topic ${topic_name} already has MRC configs. Skipping"
else
echo "topic ${topic_name} does not have MRC configs. Update necessary."
echo "Adding to execution list."
echo ${OUTPUT_TOPIC_NAME} >> ${EXECUTION_TOPIC_FILE_NAME}
fi
done
echo "============================================================================"
echo "The output of this step is available in ${EXECUTION_TOPIC_FILE_NAME}"
echo "Validation complete. Check and remove the topics that you would not like to enable for MRC from the file."
echo "============================================================================"
read -p "Press enter to continue once you have validated the file and are happy to proceed further."
echo "============================================================================"
cat ${EXECUTION_TOPIC_FILE_NAME} | while read topic_name
do
echo "Working on ${topic_name}"
if [[ -z "${SCRIPT_COMMAND_CONFIG}" ]]
then
echo "Executing without Command Config"
kafka-configs --bootstrap-server ${SCRIPT_BOOTSTRAP_SERVERS} --entity-type topics --entity-name ${topic_name} --alter --replica-placement ${SCRIPT_REPLICA_PLACEMENT_FILE_NAME}
else
echo "Executing with Command Config"
kafka-configs --bootstrap-server ${SCRIPT_BOOTSTRAP_SERVERS} --command-config ${SCRIPT_COMMAND_CONFIG} --entity-type topics --entity-name ${topic_name} --alter --replica-placement ${SCRIPT_REPLICA_PLACEMENT_FILE_NAME}
fi
echo "============================================"
done
echo "============================================================================"
echo "Done"
echo "============================================================================"
echo "Do you want me to describe all the topics for sanity [y/n]?"
read SCRIPT_DESCRIBE_READ
if [[ "$SCRIPT_DESCRIBE_READ" == "y" ]] || [[ "$SCRIPT_DESCRIBE_READ" == "Y" ]] || [[ "$SCRIPT_DESCRIBE_READ" == "yes" ]] || [[ "$SCRIPT_DESCRIBE_READ" == "YES" ]]
then
cat ${SCRIPT_TOPIC_FILE_NAME} | while read topic_name
do
if [[ -z "${SCRIPT_COMMAND_CONFIG}" ]]
then
echo "Executing without Command Config"
kafka-topics --bootstrap-server ${SCRIPT_BOOTSTRAP_SERVERS} --topic ${topic_name} --describe
else
echo "Executing with Command Config"
kafka-topics --bootstrap-server ${SCRIPT_BOOTSTRAP_SERVERS} --command-config ${SCRIPT_COMMAND_CONFIG} --topic ${topic_name} --describe
fi
echo "============================================"
done
fi
rm ${SCRIPT_TOPIC_FILE_NAME} ${EXECUTION_TOPIC_FILE_NAME}