{"push_id":912645613,"size":21,"distinct_size":17,"ref":"refs/heads/master","head":"61313bbcd78864c72c8295481f1470971cf3c62d","before":"587c706e78d13dd28832d4c209454b7ea20cb3ec","commits":[{"sha":"f82b2756f540e51b8aec4c009ea155f41108c8e5","author":{"email":"a51dda7c7ff50b61eaea0444371f4a6a9301e501@voicebase.com","name":"jbocharov"},"message":"Add parsing of config file argument and sample config file","distinct":false,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/f82b2756f540e51b8aec4c009ea155f41108c8e5"},{"sha":"1b9021b614eaae5bb31e0ced635e458fa03be14c","author":{"email":"a51dda7c7ff50b61eaea0444371f4a6a9301e501@voicebase.com","name":"jbocharov"},"message":"Re-factor DataIngest as Python class","distinct":false,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/1b9021b614eaae5bb31e0ced635e458fa03be14c"},{"sha":"0d0a0bc6806c63dda079393462576a1ebf121ff1","author":{"email":"a51dda7c7ff50b61eaea0444371f4a6a9301e501@voicebase.com","name":"jbocharov"},"message":"Generator-based Twitter ingest prototype (currently limited to a few entries)","distinct":false,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/0d0a0bc6806c63dda079393462576a1ebf121ff1"},{"sha":"fea00e48153f6b7089575c98de1a39f69bed1964","author":{"email":"5b4a9e08555d97c5e8bc2fb4f3a89ddde501f7c8@trixandtrax.com","name":"Alejandro J. Rojas"},"message":"first committ\n\nNeed to assess if we need the data in a more granular format separated\nby comments, likes, shared post, permalink, id","distinct":false,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/fea00e48153f6b7089575c98de1a39f69bed1964"},{"sha":"1dccca6708f872085d75fd745b8f7197507671d9","author":{"email":"13199e3a7635fa9e2d2f2694574d322be833f696@gmail.com","name":"judofighter25"},"message":"add youtube","distinct":true,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/1dccca6708f872085d75fd745b8f7197507671d9"},{"sha":"e3a58e5b313b8e56c7658186fff7fca00a174acf","author":{"email":"13199e3a7635fa9e2d2f2694574d322be833f696@gmail.com","name":"judofighter25"},"message":"add youtube","distinct":true,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/e3a58e5b313b8e56c7658186fff7fca00a174acf"},{"sha":"d53f0c05655abee7835b87e5a112e5fc94e4dd86","author":{"email":"13199e3a7635fa9e2d2f2694574d322be833f696@gmail.com","name":"judofighter25"},"message":"Merge pull request #1 from W205-Social-Media/master\n\nMaster","distinct":true,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/d53f0c05655abee7835b87e5a112e5fc94e4dd86"},{"sha":"65c84b3f73f56ac8bae370bf3c7737d03429181d","author":{"email":"a51dda7c7ff50b61eaea0444371f4a6a9301e501@voicebase.com","name":"jbocharov"},"message":"Toy-ish implementation of S3 sink","distinct":true,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/65c84b3f73f56ac8bae370bf3c7737d03429181d"},{"sha":"f3135dba6e4c0d924f6c573b6726f8dfe85705e7","author":{"email":"75e77f44bac6cbb231c20c7f0689d984d6109366@users.noreply.github.com","name":"jbocharov-mids"},"message":"Merge pull request #2 from venamax/develop\n\nfirst committ","distinct":true,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/f3135dba6e4c0d924f6c573b6726f8dfe85705e7"},{"sha":"bc1a537c9684818231c600c72d2383c5ed7cb913","author":{"email":"a51dda7c7ff50b61eaea0444371f4a6a9301e501@voicebase.com","name":"jbocharov"},"message":"Fix Data ingest and S3 sink bugs - working update in batches to S3","distinct":true,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/bc1a537c9684818231c600c72d2383c5ed7cb913"},{"sha":"26f4a66e0b254134a11833464b2d8abed4bbb1cd","author":{"email":"a51dda7c7ff50b61eaea0444371f4a6a9301e501@voicebase.com","name":"jbocharov"},"message":"Switch batch format to JSON, added data sample","distinct":true,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/26f4a66e0b254134a11833464b2d8abed4bbb1cd"},{"sha":"4785673f5386905aea8e4e16efab174eb200ff05","author":{"email":"a51dda7c7ff50b61eaea0444371f4a6a9301e501@voicebase.com","name":"jbocharov"},"message":"Add support for and example of local sink","distinct":true,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/4785673f5386905aea8e4e16efab174eb200ff05"},{"sha":"0d2135a8cc360f550fdb0f7889b09d9c7e126a6e","author":{"email":"a51dda7c7ff50b61eaea0444371f4a6a9301e501@voicebase.com","name":"jbocharov"},"message":"Fix example config to use aws_access_key_id","distinct":true,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/0d2135a8cc360f550fdb0f7889b09d9c7e126a6e"},{"sha":"100db760c671a3ff2b96eb82796e0cd581dbb48e","author":{"email":"a51dda7c7ff50b61eaea0444371f4a6a9301e501@voicebase.com","name":"jbocharov"},"message":"Update Twitter to use streaming api, write progress to stdout like .........| (. for record, | for batch)","distinct":true,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/100db760c671a3ff2b96eb82796e0cd581dbb48e"},{"sha":"373d9cc265c50eff41edf105fb38045dbc6f7f84","author":{"email":"a51dda7c7ff50b61eaea0444371f4a6a9301e501@voicebase.com","name":"jbocharov"},"message":"Update config examples with batch_size and track parameters","distinct":true,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/373d9cc265c50eff41edf105fb38045dbc6f7f84"},{"sha":"628b2688e1f99105766cb308d0b761dd425f347b","author":{"email":"a51dda7c7ff50b61eaea0444371f4a6a9301e501@voicebase.com","name":"jbocharov"},"message":"Fix S3 sink to use UTC (GMT +0) times","distinct":true,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/628b2688e1f99105766cb308d0b761dd425f347b"},{"sha":"ae4f144ea9256b4b53fe497a656be38f32213277","author":{"email":"a51dda7c7ff50b61eaea0444371f4a6a9301e501@voicebase.com","name":"jbocharov"},"message":"Fix tweet formatting issues when streaming from twitter","distinct":true,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/ae4f144ea9256b4b53fe497a656be38f32213277"},{"sha":"e8e3a4c8ce69593d6341353a02676919d9cf729d","author":{"email":"a51dda7c7ff50b61eaea0444371f4a6a9301e501@voicebase.com","name":"jbocharov"},"message":"Added S3 Data Ingest Source support, S3 Socket Sink, which still prints to stdout","distinct":true,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/e8e3a4c8ce69593d6341353a02676919d9cf729d"},{"sha":"9a39a49435d6dacc8f7c54c23af48e144c50efe8","author":{"email":"a51dda7c7ff50b61eaea0444371f4a6a9301e501@voicebase.com","name":"jbocharov"},"message":"Added working Local sample -> Postgres ingest (verbose logging): python DataIngest.py --config .w205-postgres-sample.cfg","distinct":true,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/9a39a49435d6dacc8f7c54c23af48e144c50efe8"},{"sha":"e5c17272bebef93238fc8cd0f579e543b307bb04","author":{"email":"a51dda7c7ff50b61eaea0444371f4a6a9301e501@voicebase.com","name":"jbocharov"},"message":"Ability to read and write small and large tweet samples to Postgres, including same samples. To use:\n\nunzip -d ../small-sample small-sample.zip\nunzip -d ../large-sample large_sample.zip\n\npython DataIngest.py --config small-sample-postgres.cfg\npython DataIngest.py --config large-sample-postgres.cfg","distinct":true,"url":"https://api.github.com/repos/venamax/w205-data-ingest/commits/e5c17272bebef93238fc8cd0f579e543b307bb04"}]}