From f43564638f8c044a1d6f47109571a6ced1c65e30 Mon Sep 17 00:00:00 2001 From: zimrideops Date: Tue, 20 Dec 2016 05:34:27 -0800 Subject: [PATCH] weekly opsbase update (#45) Author: none **Files changed (97):** base/ops/config/modules/pillar/new_services_pillar.py (+0 -24 ~24) base/ops/config/modules/pillar/v2_services.py (+0 -118 ~118) base/ops/config/modules/pillar/workflow_repositories.py (+0 -16 ~16) base/ops/config/modules/states/boto_dynamodb.py (+23 -6 ~29) base/ops/config/pillar/deleted_iam_objects.sls (+1 -0 ~1) base/ops/config/pillar/github_teams.sls (+18 -0 ~18) base/ops/config/pillar/groups.sls (+81 -46 ~127) base/ops/config/pillar/lyft_users.sls (+92 -55 ~147) base/ops/config/pillar/nutcracker.sls (+25 -12 ~37) base/ops/config/pillar/policies.sls (+40 -0 ~40) base/ops/config/pillar/roles.sls (+21 -0 ~21) base/ops/config/pillar/services.sls (+20 -44 ~64) base/ops/config/states/base/logagent/hekad.toml (+1 -1 ~2) base/ops/config/states/base/manage_install.sls (+0 -19 ~19) base/ops/config/states/base/pulldeploy/check.py (+7 -4 ~11) base/ops/config/states/base/pulldeploy/pulldeploy.py (+24 -17 ~41) base/ops/config/states/base/salt/new_services_pillar.conf (+0 -2 ~2) base/ops/config/states/base/services.json (+0 -1 ~1) base/ops/config/states/envoy2/init.sls (+3 -3 ~6) base/ops/config/states/lyft-python/gunicorn/gunicorn.conf (+3 -2 ~5) base/ops/config/states/lyft-python/init.sls (+1 -1 ~2) base/ops/orca/iam_roles.sls (+6 -0 ~6) base/ops/orca/iam_users.sls (+0 -2 ~2) base/tests/test_new_services_pillar.py (+0 -66 ~66) base/tests/test_pulldeploy.py (+184 -0 ~184) base/tests/test_pulldeploy_check.py (+103 -6 ~109) bootartifact/Dockerfile (+1 -1 ~2) buildnotify/Dockerfile (+1 -1 ~2) buildnotify/Makefile (+1 -1 ~2) buildnotify/buildnotify/ciexec.py (+9 -5 ~14) buildnotify/buildnotify/cli.py (+1 -0 ~1) buildnotify/buildnotify/delete_old_branches.py (+1 -0 ~1) buildnotify/buildnotify/github.py (+1 -1 ~2) buildnotify/buildnotify/routes.py (+8 -2 ~10) buildnotify/buildnotify/services.py (+23 -16 ~39) buildnotify/buildnotify/settings.py (+7 -0 ~7) buildnotify/buildnotify/submit_queue.py (+28 -14 ~42) buildnotify/buildnotify/tests/test_ciexec.py (+16 -11 ~27) buildnotify/buildnotify/tests/test_jobs.py (+2 -2 ~4) buildnotify/buildnotify/tests/test_routes.py (+10 -2 ~12) buildnotify/buildnotify/tests/test_services.py (+565 -473 ~1038) buildnotify/buildnotify/tests/test_submit_queue.py (+1 -0 ~1) buildnotify/buildnotify/tests/test_v3_callbacks.py (+7 -3 ~10) buildnotify/buildnotify/tests/test_v3_jenkins_jobs.py (+4 -1 ~5) buildnotify/buildnotify/tests/test_v3_models.py (+3 -3 ~6) buildnotify/buildnotify/v2_services.py (+27 -24 ~51) buildnotify/buildnotify/v3_callbacks.py (+4 -2 ~6) buildnotify/buildnotify/v3_jenkins.py (+4 -4 ~8) buildnotify/buildnotify/v3_jenkins_jobs.py (+2 -1 ~3) buildnotify/buildnotify/v3_manifests.py (+3 -6 ~9) buildnotify/buildnotify/v3_models.py (+2 -2 ~4) buildnotify/ops/config/pillar/buildnotify.sls (+2 -0 ~2) buildnotify/requirements.txt (+25 -24 ~49) buildnotify/scripts/prfairy.py (+1 -0 ~1) buildslave/Makefile (+3 -0 ~3) buildslave/deploy (+1 -1 ~2) buildslave/ops/orca/buildslave.sls (+1 -1 ~2) docker/onebox/starter (+2 -2 ~4) docs/datastores-network/oncall-runbook.md (+2 -0 ~2) docs/mongodb/runbook.md (+58 -0 ~58) docs/redis/overview.md (+5 -9 ~14) gateway/Dockerfile (+1 -1 ~2) gateway/manifest.yaml (+1 -1 ~2) gateway/ops/config/states/gateway/attach-elastic-ip.sh (+41 -0 ~41) gateway/ops/config/states/gateway/init.sls (+16 -0 ~16) gateway/ops/orca/gateway.sls (+12 -0 ~12) grafana/Dockerfile (+1 -1 ~2) hacktools/fabfile.py (+1 -1 ~2) jenkins/apitestrevisions/force_update_last_good_revision.py (+6 -0 ~6) jenkins/apitestrevisions/get_last_good_revision.py (+5 -1 ~6) jenkins/apitestrevisions/send_failure_report.py (+17 -7 ~24) jenkins/apitestrevisions/store_test_failure_report.py (+6 -1 ~7) jenkins/apitestrevisions/tests/test_send_failure_report.py (+9 -0 ~9) jenkins/docs/runbook.md (+9 -8 ~17) jenkins/ops/config/states/jenkins/init.sls (+16 -0 ~16) jenkins/ops/config/states/jenkins/templates/test-dummy.xml (+71 -0 ~71) jenkins/ops/config/states/jenkins/templates/test_image.xml (+30 -16 ~46) jenkins/requirements.txt (+1 -1 ~2) opstools/Dockerfile (+1 -1 ~2) opstools/cloudwatch2wavefront/cloudwatch.py (+6 -0 ~6) opstools/dynamodb_restore/streams_to_dynamodb_restore_tool/streams_dynamodb_processor.py (+1 -0 ~1) orca/Dockerfile (+1 -1 ~2) orca/pillar/aws.sls (+5 -4 ~9) redis/ops/orca/redis.sls (+1 -1 ~2) statsd/Dockerfile (+1 -1 ~2) statsd/docs/runbook.md (+3 -3 ~6) statsd/ops/config/pillar/env.sls (+4 -0 ~4) statsd/ops/config/pillar/top.sls (+5 -0 ~5) statsd/ops/config/states/statsdbatch/init.sls (+3 -0 ~3) statsd/ops/config/states/top.sls (+8 -0 ~8) statsd/ops/orca/dynamo.sls (+16 -0 ~16) statsd/ops/orca/statsd.sls (+68 -4 ~72) statsd/requirements.txt (+31 -0 ~31) statsd/scripts/__init__.py (+0 -0 ~0) statsd/scripts/config-generator.py (+173 -0 ~173) vpcproxy/Dockerfile (+1 -1 ~2) vpn/requirements.txt (+0 -1 ~1) **104 commits:** [Fix failure report script (#9486)](https://github.com/lyft/ops/commit/f5249ea17853de935f28b96c8611f875b8fcd0f4) [weekly opsbase update (#9489)](https://github.com/lyft/ops/commit/4143c0260cf49fb847ce05dc5e1862b5d52e5fbf) [weekly opsbase update (#9490)](https://github.com/lyft/ops/commit/29e9f6b3ac81ea629242dcea7f36c59c01d20da9) [weekly opsbase update (#9491)](https://github.com/lyft/ops/commit/88885fd8dfa02b7e81a74720ed3b5a06bea9f59a) [weekly opsbase update (#9492)](https://github.com/lyft/ops/commit/e911c09ee1349f123563b3944431367ebf1f89e2) [weekly opsbase update (#9493)](https://github.com/lyft/ops/commit/4a1dd0b369ae5ce0dc31499489062cc3815404ed) [weekly opsbase update (#9494)](https://github.com/lyft/ops/commit/6349f052f4390b02de4da2ce75e62a388ab0b50c) [weekly opsbase update (#9495)](https://github.com/lyft/ops/commit/127d2f6661bd74632319a6f224eb1383f060c2ed) [weekly opsbase update (#9496)](https://github.com/lyft/ops/commit/16bc29f5be26d33cc3b4b4452826405617ea7f70) [PROV-796: Check.py fails with shadow traffic clusters (#9487)](https://github.com/lyft/ops/commit/b50361464ce177b2dfde2fac7e3bbf6ecdf44851) [updated some destructive permissions (#9484)](https://github.com/lyft/ops/commit/aa8749401b5587c830d10dad791b065be51a7036) [DS-550: scale eventdedupe redis (#9499)](https://github.com/lyft/ops/commit/876a387ef4bbfc9c15a7745420675019f92cd023) [add ridesapi slack channel (#9483)](https://github.com/lyft/ops/commit/a0972c696f33b1bf44d3c81d2c6542d06b427766) [Backport features (#9500)](https://github.com/lyft/ops/commit/a076e871f8d66210dae77f9be8a6a4eea1ade7de) [redis/docs: misc improvements (#9505)](https://github.com/lyft/ops/commit/cfcf7ebc8b5146ef60040ed9c44bc0302d581436) [hacktools: fix nutcracker_sls typo (#9507)](https://github.com/lyft/ops/commit/6e77e1c823a98780b100daebde422f9e025c195e) [Use a version of py-googleauth that does not pin deps (#9503)](https://github.com/lyft/ops/commit/aaf00167c4f2abcae0925a8fa0ab620355f58862) [[SECHELP-160] Add mpaine to AWS (#9502)](https://github.com/lyft/ops/commit/6c2d40ad65869f4d52b9dbeba4692855b3f6d0f1) [updaing manifest to be in parity with services.sls (#9510)](https://github.com/lyft/ops/commit/856d1d18c8ec9fdeca05a171fc5909cdbf29b967) [nutcracker: update eventdedupe config (#9504)](https://github.com/lyft/ops/commit/d5c19be7cd80b82c5fa0721a2af76539b257623e) [Give myself keys to the kingdom (#9508)](https://github.com/lyft/ops/commit/c6f3014b35333a41b09268e2d207db1e34d498fc) [[SECHELP-135] Add brady to ats/images admin (#9509)](https://github.com/lyft/ops/commit/123896194fcf3df982814e6809956fa0b56fd06f) [Update runbook for buildnotify localdev to v3 devbox instructions. (#9512)](https://github.com/lyft/ops/commit/4b7695de86fd48496f6c8cb2a006cbd467172db2) [[SEC-1149] Add lyft-data-sience-s3-access policy to Developers and Data_Platform. (#9498)](https://github.com/lyft/ops/commit/c237301b371a4e3d7d19d47041a5348909821ca6) [Use manifest to determine upstream tests. (#9513)](https://github.com/lyft/ops/commit/47a4c8c5868386dba6713748c192a9220acce615) [Add ci_version to Dynamo tables (#9515)](https://github.com/lyft/ops/commit/6b19a3c5ea0402225ac981ab3b5cbe93dad2bae6) [TSDB archive service: just lots of S3 (#9517)](https://github.com/lyft/ops/commit/59d816089938ceba8a05c4161e0825665e83f571) [jenkins: fix curl to upload console to s3 (#9519)](https://github.com/lyft/ops/commit/95e7eac019a6da54d54a93926941a67a877d83d8) [Fix vincentchang's unix name. (#9514)](https://github.com/lyft/ops/commit/872203e7f3c91e8cc042ad46114703263b726878) [Fix log rotation in brochure service (#9518)](https://github.com/lyft/ops/commit/ccd380b5cbbafe2ce2fb3c0e479b82425830fdd3) [envoy prod 773ff47816c63327896c4eea8fefce78e206b2b5 (#9524)](https://github.com/lyft/ops/commit/789621259656887da117a135d4bb277fa7e36f9c) [Revert new services pillar (#9521)](https://github.com/lyft/ops/commit/e67dd981b398203fb83fb6a2406df7819c56a054) [[SECHELP-163] Add bnorquist to snitch ro group (#9511)](https://github.com/lyft/ops/commit/88d7dacaf91b5318a5f00cc4f36d3f8ef12ee57e) [Update lyft_users.sls](https://github.com/lyft/ops/commit/f03691b22be5192a60335623cbc408239048228b) [Removed inactive developers (#9506)](https://github.com/lyft/ops/commit/c117f88f69d5ecc1da036d4875cb96b071777702) [Include timeout in GeventWorkerTimeoutException message (#9501)](https://github.com/lyft/ops/commit/da7fe71fecbe05c4c04a0313418400d80f97fe2a) [[SEC-727] add attach elastic ip script (#9456)](https://github.com/lyft/ops/commit/2a1d83c628889b837a0b4b6fcce239428c582c8a) [buildnotify: post to ciexec with json body (#9527)](https://github.com/lyft/ops/commit/ac4fb35c575b2864114425fd65486de1f6f99c36) [fixed typo (#9529)](https://github.com/lyft/ops/commit/2f36c755ec45360485032ef77e63a80e5a2cc69a) [Call build with parameters for dev (#9530)](https://github.com/lyft/ops/commit/16ecfed10d2f71923a8f9331355f7006efcfd64e) [link to streaming schedule (#9531)](https://github.com/lyft/ops/commit/ef2e7db1a7144c0bc3693f2c408805fe494c56b7) [DS-438 - mongodb runbook - document maxconns problem (#9522)](https://github.com/lyft/ops/commit/0b2554f420c1897c178de1daf4ef026ded062c22) [move elastic ip file location (#9533)](https://github.com/lyft/ops/commit/18b8f9272f2096b020b7c1eb9de7a6c05c42de8d) [jenkins: upgrade testrunner to include links (#9534)](https://github.com/lyft/ops/commit/1f8711ba8975dd7b4154e6d3c0a576ec3da2e2bf) [Remove upstream_tests definition for supplyalerts from services.sls (#9520)](https://github.com/lyft/ops/commit/2d22abd4925a34ca528874c2831a7a0e84b7a627) [jenkins: put console output at the correct place for archival (#9536)](https://github.com/lyft/ops/commit/79ac04ab010b65c12c14617dea8dda0a297caa8b) [jenkins: pick up the correct test runner :( (#9537)](https://github.com/lyft/ops/commit/ddae4731b4f722c4b2219b0dff20b70edddd606b) [opstools: dynamo streams restore add finish message (#9539)](https://github.com/lyft/ops/commit/4ef0b701baabad1987e0db2f1640da7744004b16) [Added Raman Malik (Marketing) (#9525)](https://github.com/lyft/ops/commit/3e7a4004a86a62c2b8cecb17a5fc5d774f0637e0) [Trigger V3 dev jobs with parameters (#9532)](https://github.com/lyft/ops/commit/3c7b60a742bb0b7ac30c2df6734c03e25af6e28b) [log whether or not we are using manifest for tests - because I am at my wits' end (#9541)](https://github.com/lyft/ops/commit/b350805b396ec2f876c460cefb7c728b8622e3ba) [Remove upstream definitions for api from services.sls (#9544)](https://github.com/lyft/ops/commit/a2969d7b6ac2b8e4c9cb565f7071b0900c50def8) [Revert "log whether or not we are using manifest for tests - because (#9541)" (#9546)](https://github.com/lyft/ops/commit/03c2ce8c3accf1211f6c167a5aacfa948ed6c16b) [Remove "set -e" (#9538)](https://github.com/lyft/ops/commit/daac6a81814384ed525e2763ed174166fecce278) [fixed typo (#9548)](https://github.com/lyft/ops/commit/b8f0b133ff608148ef3b40a61ccb20896902bf8d) [Fixed another problem with checking exit codes with set -e (#9549)](https://github.com/lyft/ops/commit/5b04e3cee34d2bc5094637cbae0aa955ae86ca71) [Allow runtime override to skip running V2 PR tests. (#9458)](https://github.com/lyft/ops/commit/6c3dbe0f234b1438194b9c2579940a7959403d6e) [envoy deploy (#9551)](https://github.com/lyft/ops/commit/2b4c4447e73bcb21781330fb19a0f5c877df728f) [Automation 1435 incentives float dep (#9526)](https://github.com/lyft/ops/commit/8de4ed91e8717018441440a8fbaf4299a079ac9a) [analyticsingestloadtester cannot deploy with new instances (#9555)](https://github.com/lyft/ops/commit/87ccc1ac3d42d007d20863bdcee2bd45e18b0ced) [Update statsd dashboard links (#9556)](https://github.com/lyft/ops/commit/35aa09c9bba9ece74bbad561690d71bb6dc70888) [Added sfo eips (#9554)](https://github.com/lyft/ops/commit/82ffcd9d532715c1bdba7b803a756dcd38aabeb9) [buildnotify: stringify all values for ciexec (#9559)](https://github.com/lyft/ops/commit/6019e69d9dce0aa549173e8666c9bf2b0bad8a8c) [fixed a typo (#9561)](https://github.com/lyft/ops/commit/9430a905a3766fd1cb466ff1e7b581834ba5e626) [Give redshift-production-iad access to lyft-analytics and lyft-analytics-tmp S3 buckets. (#9552)](https://github.com/lyft/ops/commit/b83b50f7135a146b0ff6e51c186f7f005025543d) [get region from az (#9562)](https://github.com/lyft/ops/commit/c40aaea5671a00a012d272affeb7beb0befb4be0) [Fix high CPU alarms on buildnotify (#9563)](https://github.com/lyft/ops/commit/30bd6821c3d30483ea6d7a2a0ed13073c5ddeda7) [jenkins: factor in https://github.com/lyft/testrunner/pull/328/ (#9567)](https://github.com/lyft/ops/commit/175ea962f9ce20d2dfc2afb88e45f2cf0b2dbc58) [Add moar types to Buildnotify. (#9566)](https://github.com/lyft/ops/commit/f82feae2439a5899588d367fd7b79dd8f5afc0cf) [Tweak link description (#9557)](https://github.com/lyft/ops/commit/0e378104cb1f8e7843ecdf6bc4d590d28df3dcfe) [Give redshift-production-iad read access to the eta-models buckets. (#9568)](https://github.com/lyft/ops/commit/2f2274bf4bb3d597dfea38bc639b70fba616deb8) [Report status on V3 Dev jobs to Buildnotify (#9570)](https://github.com/lyft/ops/commit/4e5d0480320392c03e51d94b8fe8b12d2968fe62) [Remove references to devbox-default (#9571)](https://github.com/lyft/ops/commit/fdba6a29e48aef433b02263ed7d024ff3a340550) [envoy 551b8dc38ec0ed31e9eb555792da73da4672e747 deploy. (#9574)](https://github.com/lyft/ops/commit/ecc8c035b113eca26c3f2a23afefd5164c1d4eb1) [Update requirements for buildnotify (#9573)](https://github.com/lyft/ops/commit/c56e8a826aef3e99c52f51cbb18a48386046480a) [Envoy 37e6e5b5224e18f27755f4751440e6879536a7ac (#9576)](https://github.com/lyft/ops/commit/d2bcc798e30b5a0836df10810fff1d8a37e97562) [Don't run Skipped V2 tests when pushing to master. (#9569)](https://github.com/lyft/ops/commit/4c0fef6734f0755a8cc25e78a0b31a3a3ec075c4) [Give S3 access back to Analytics group. (#9578)](https://github.com/lyft/ops/commit/f752b2aa87685c62f4f147302950237efceebfaa) [Fix a hole in our MFA device + AccessKey self-service policy. (#9575)](https://github.com/lyft/ops/commit/2107678873239d71456d4e11ffc39357345c80b8) [[SECHELP-174] Erin Gray to github (#9565)](https://github.com/lyft/ops/commit/92cfc98c2f4e189d91942342c240fababc53577f) [[SECHELP-180] Add Jerrison Li to analyst db (#9580)](https://github.com/lyft/ops/commit/58800d5aa0fe0d9b82310739ded0b5df3f10c2d3) [Factor out the v2_services logic that updates the upstream_tests map. (#9579)](https://github.com/lyft/ops/commit/97a86a684e21535a8df38ec58a8bffb8dbe0b42b) [[SECHELP-166] Add Scott Owades to guardian, ridehistory, runtime ro r (#9582)](https://github.com/lyft/ops/commit/02baa05ede44281cb62f640a9de24868f6dcc323) [Get off slackclient v1.0.4 - I think it kills deploy_bot.](https://github.com/lyft/ops/commit/b25e39fa2a866a334fe839245f5da11631b2ce8c) [Always hit v3_manifests when finding upstream tests. (#9585)](https://github.com/lyft/ops/commit/617589aa3c68c4396bc5366d6cb11e4fadc4db17) [Fix the handling of limited_test_set in v2_services.py (#9586)](https://github.com/lyft/ops/commit/73c557e56460bd5d589a965af621b72fef330ace) [offboard jthai (#9587)](https://github.com/lyft/ops/commit/48291b4fcdd1f9b127c63b5c4a35bc8e76f57a03) [statsd batch box (#9528)](https://github.com/lyft/ops/commit/f1a06c29ea78d980ebdf3545e061ac104e7b5245) [grant security + data privacy ssh access to team (#9588)](https://github.com/lyft/ops/commit/066ed9caf553d3d8f7103a311365bbaf32e4bacb) [[SECHELP-167] Give amartine read access to requested repos (#9589)](https://github.com/lyft/ops/commit/ac496919a704da631a6acbf35770911baf300308) [Move Lift*MFADevices out of MFA-protected action, as we use them to bootstrap MFA in blessclient and awsaccess. (#9591)](https://github.com/lyft/ops/commit/866b54105f63628118bf6dcd5f9c142abd9785e6) [Update envoy sha (#9592)](https://github.com/lyft/ops/commit/fcffea685d59391696aa0a43fb1a7262332f1c6f) [Add dynamo table where we store failure reports (#9595)](https://github.com/lyft/ops/commit/e0d2d37876b2d7a77767537c868036ff9a7fb7ab) [Default commits to an empty map (#9596)](https://github.com/lyft/ops/commit/d3cbeca525246b58a1cfad6c661d40ac40637686) [[SECHELP-179] Add Jordeen Chang to financial repo (#9564)](https://github.com/lyft/ops/commit/d523bb08305d3ee86f43beeeac8b8df0f4226797) [jenkins: add a test dummy job (#9597)](https://github.com/lyft/ops/commit/4086573f22fae94d7c94e55e8aa408629631ca2b) [Update envoy sha (#9598)](https://github.com/lyft/ops/commit/995d8194a3b48406627b6207d7be7e812f926857) [Block mongo logs from Envoy from entering heka (#9599)](https://github.com/lyft/ops/commit/ac4c1f6409fef5553ef1ec8cbbb8fe2a6dd14e11) [use v3 tests for datatoolkit (#9581)](https://github.com/lyft/ops/commit/091621d0a73e211c4c454289665adbde5ee612d5) [Simplifying the upstream test process](https://github.com/lyft/ops/commit/08b6c30d7460c1b0ba346eda841234b3a9062c7f) [updated iam groups to expand desctructive actions (#9603)](https://github.com/lyft/ops/commit/a53ceb2496054df7641bf7c6acc7a2d93fd72f15) [Add TCS dependency to integration tests (#9594)](https://github.com/lyft/ops/commit/1b47618e22ae71d9e8e0966b3da5bbc1f703e59a) [Add ability to override region name while creating alarms (#9560)](https://github.com/lyft/ops/commit/4bd80469ee47dd969f61c507f7d41b9a5b8fbca9) [Add clientbuilds development deploy step (#9606)](https://github.com/lyft/ops/commit/d4a47bd56ccb8034b729de3ac4c1532cd1295700) NOT PASSING TESTS --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f4f7edc..c9d24e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM lyft/opsbase:d883c38f9da3a29967651e683cb97518c2c484f8 +FROM lyft/opsbase:d4a47bd56ccb8034b729de3ac4c1532cd1295700 COPY . /code/statsrelay-private RUN apt-get update -y && \ apt-get install -y software-properties-common python-software-properties && \