[{"data":1,"prerenderedAt":208},["ShallowReactive",2],{"slug-content-en-\u002Fprojects\u002Fdb-crawler":3},{"kind":4,"item":5,"categories":59},"project",{"id":6,"title":7,"body":8,"dateEnd":38,"dateStart":39,"description":40,"extension":41,"featuredInCv":42,"link":43,"meta":44,"navigation":45,"path":46,"repo":43,"seo":47,"stem":48,"tags":49,"__hash__":58},"projects_en\u002Fprojects\u002Fdb-crawler.md","db-crawler",{"type":9,"value":10,"toc":33},"minimark",[11,16],[12,13,15],"h2",{"id":14},"about-the-project","About the Project",[17,18,19,20,23,24,26,27,29,30,32],"p",{},"The db-crawler is a crawler for Deutsche Bahn's public interfaces, built as a self-coordinated test project during a mandatory internship.",[21,22],"br",{},"\nEvery morning shortly after midnight, the tool wrote all of Deutsche Bahn's scheduled timetable data into a database. Since this had to happen separately for every hour and every train station across Germany, several cloud servers were provisioned on demand via the public Hetzner API, booted up, and automatically configured via cloud-init.",[21,25],{},"\nAs soon as a given server was fully set up, it independently picked up a previously and automatically assigned work package (a list of stations) and started fetching all scheduled data for them.",[21,28],{},"\nThe data was sent back to the master server and stored there in a relational database.",[21,31],{},"\nOnce the work package was completed, the server shut itself down and automatically deleted the resource, avoiding unnecessary costs.",{"title":34,"searchDepth":35,"depth":35,"links":36},"",2,[37],{"id":14,"depth":35,"text":15},"2019-05","2019-04","A crawler for Deutsche Bahn's public interfaces.","md",false,null,{},true,"\u002Fprojects\u002Fdb-crawler",{"title":7,"description":40},"projects\u002Fdb-crawler",[50,51,52,53,54,55,56,57],"rest","docker","rabbitmq","sql","git","cicd","terraform","python","l7o_HsFIq2ZAV1DnBuR40i6mCn_UARDlHMtYS33BdKU",[60,68,89,110,140,152,160,169],{"name":61,"items":62},"Databases",[63,65],{"slug":53,"name":64},"SQL (MySQL, MariaDB)",{"slug":66,"name":67},"sqlite","SQLite",{"name":69,"items":70},"Frontend",[71,74,77,80,83,86],{"slug":72,"name":73},"html5","HTML5",{"slug":75,"name":76},"css3","CSS3",{"slug":78,"name":79},"tailwindcss","Tailwind CSS",{"slug":81,"name":82},"bootstrap","Bootstrap",{"slug":84,"name":85},"uikit","UIkit",{"slug":87,"name":88},"extjs","Ext JS",{"name":90,"items":91},"JavaScript Ecosystem",[92,95,98,101,104,107],{"slug":93,"name":94},"javascript","JavaScript",{"slug":96,"name":97},"typescript","TypeScript",{"slug":99,"name":100},"nodejs","Node.js",{"slug":102,"name":103},"vuejs","Vue.js",{"slug":105,"name":106},"nuxt","Nuxt.JS",{"slug":108,"name":109},"deno","Deno",{"name":111,"items":112},"Backend",[113,116,119,122,125,128,131,134,137],{"slug":114,"name":115},"php","PHP",{"slug":117,"name":118},"laravel","Laravel",{"slug":120,"name":121},"symfony","Symfony",{"slug":123,"name":124},"laminas","Laminas",{"slug":126,"name":127},"mezzio","Mezzio",{"slug":129,"name":130},"zend","Zend Framework",{"slug":132,"name":133},"wordpress","WordPress",{"slug":135,"name":136},"shopware","Shopware",{"slug":138,"name":139},"spryker","Spryker",{"name":141,"items":142},"Mobile",[143,146,149],{"slug":144,"name":145},"dart","Dart",{"slug":147,"name":148},"flutter","Flutter",{"slug":150,"name":151},"android","Android",{"name":153,"items":154},"Other Languages",[155,158],{"slug":156,"name":157},"java","Java",{"slug":57,"name":159},"Python",{"name":161,"items":162},"Operating Systems",[163,166],{"slug":164,"name":165},"linux","Linux",{"slug":167,"name":168},"windows","Windows",{"name":170,"items":171},"Other",[172,174,176,179,182,185,188,191,194,197,199,202,204,206],{"slug":54,"name":173},"Git",{"slug":51,"name":175},"Docker",{"slug":177,"name":178},"jenkins","Jenkins",{"slug":180,"name":181},"jvm","JVM",{"slug":183,"name":184},"aws","AWS",{"slug":186,"name":187},"cloudflare","Cloudflare",{"slug":189,"name":190},"nginx","Nginx",{"slug":192,"name":193},"redis","Redis",{"slug":195,"name":196},"networking","Networking",{"slug":52,"name":198},"RabbitMQ",{"slug":200,"name":201},"oidc","OIDC",{"slug":50,"name":203},"REST",{"slug":56,"name":205},"Terraform",{"slug":55,"name":207},"CI\u002FCD",1782416851959]