{"action":"created","comment":{"url":"https://api.github.com/repos/CommBank/maestro/pulls/comments/27366954","id":27366954,"diff_hunk":"@@ -146,29 +228,42 @@ trait UploadExecution {\n object UploadEx {\n   val logger = Logger.getLogger(\"Upload\")\n \n-  def execution(conf: UploadConfig): Execution[UploadInfo] = for {\n-    _      <- Execution.from {\n-                logger.info(\"Start of upload from ${conf.localIngestPath}\")\n-                logger.info(s\"tableName        = ${conf.tablename}\")\n-                logger.info(s\"filePattern      = ${conf.filePattern}\")\n-                logger.info(s\"controlPattern   = ${conf.controlPattern}\")\n-                logger.info(s\"localIngestPath  = ${conf.localIngestPath}\")\n-                logger.info(s\"localArchivePath = ${conf.localArchivePath}\")\n-                logger.info(s\"hdfsArchivePath  = ${conf.hdfsArchivePath}\")\n-                logger.info(s\"hdfsLandingPath  = ${conf.hdfsLandingPath}\")\n-              }\n-    files  <- Execution.fromEither(Input.findFiles(\n-                conf.localIngestPath, conf.tablename, conf.filePattern, conf.controlPattern\n-              ))\n-    _      <- Execution.from(\n-                files.controlFiles.foreach(ctrl =>logger.info(s\"skipping control file ${ctrl.file.getName}\"))\n-              )\n+  /** fileMatchingExecution to extract the datafiles matched with the filePattern */\n+  def matcher(conf: UploadConfig): Execution[List[Data]] = for {\n+    _ <- Execution.from {\n+      logger.info(\"Start of FileMatching from ${conf.localIngestPath}\")\n+      logger.info(conf.toString)\n+    }\n+    files <- Execution.fromEither(Input.findFiles(\n+      conf.localIngestPath, conf.tablename, conf.filePattern, conf.controlPattern\n+    ))\n+    _ <- Execution.from(\n+      files.controlFiles.foreach(ctrl => logger.info(s\"skipping control file ${ctrl.file.getName}\"))\n+    )\n+  } yield files.dataFiles\n+\n+  /** UploadFilesExecution uploads Data */\n+  def upload(conf: UploadConfig, files: List[Data]): Execution[UploadInfo] = for {\n+    _ <- Execution.from(logger.info(\"Starting fileUploads\"))\n     copied <- Execution.fromHdfs(\n-                Push.push(files.dataFiles, conf.hdfsLandingPath, conf.localArchivePath, conf.hdfsArchivePath)\n-              )\n-    _      <- Execution.from {\n-                copied.foreach { case Copied(files, dest) => logger.info(s\"Copied ${files.map(_.getName).mkString(\", \")} to $dest \") }\n-                logger.info(s\"Upload ended from ${conf.localIngestPath}\")\n-              }\n+      Push.push(files, conf.hdfsLandingPath, conf.localArchivePath, conf.hdfsArchivePath)\n+    )\n+    _ <- Execution.from {\n+      copied.foreach { case Copied(files, dest) => logger.info(s\"Copied ${files.map(_.getName).mkString(\", \")} to $dest \")}\n+      logger.info(s\"Upload ended\")\n+    }\n   } yield UploadInfo(copied.map(_.dest.toString))\n-}\n+\n+  /** Extract Header and trailer and file and Parse same*/\n+  def parser[H, T](parseH: String => H, parseT: String => T, file: String): Result[(H, T)]= {\n+    val inputReverse       = new ReversedLinesFileReader(FileUtils.getFile(file))\n+    val inputHeader        = new BufferedReader(new FileReader(file))\n+    val trailer            = Result.ok(inputReverse.readLine())","path":"maestro-core/src/main/scala/au/com/cba/omnia/maestro/core/task/UploadExecution.scala","position":215,"original_position":215,"commit_id":"ba41bf5603544ffa63483a9416f84dc346f9bf1c","original_commit_id":"ba41bf5603544ffa63483a9416f84dc346f9bf1c","user":{"login":"SamRoberts","id":1086978,"avatar_url":"https://avatars.githubusercontent.com/u/1086978?v=3","gravatar_id":"","url":"https://api.github.com/users/SamRoberts","html_url":"https://github.com/SamRoberts","followers_url":"https://api.github.com/users/SamRoberts/followers","following_url":"https://api.github.com/users/SamRoberts/following{/other_user}","gists_url":"https://api.github.com/users/SamRoberts/gists{/gist_id}","starred_url":"https://api.github.com/users/SamRoberts/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/SamRoberts/subscriptions","organizations_url":"https://api.github.com/users/SamRoberts/orgs","repos_url":"https://api.github.com/users/SamRoberts/repos","events_url":"https://api.github.com/users/SamRoberts/events{/privacy}","received_events_url":"https://api.github.com/users/SamRoberts/received_events","type":"User","site_admin":false},"body":"Good points, @jonjuscorpion666. Ok then.","created_at":"2015-03-30T05:08:59Z","updated_at":"2015-03-30T05:08:59Z","html_url":"https://github.com/CommBank/maestro/pull/343#discussion_r27366954","pull_request_url":"https://api.github.com/repos/CommBank/maestro/pulls/343","_links":{"self":{"href":"https://api.github.com/repos/CommBank/maestro/pulls/comments/27366954"},"html":{"href":"https://github.com/CommBank/maestro/pull/343#discussion_r27366954"},"pull_request":{"href":"https://api.github.com/repos/CommBank/maestro/pulls/343"}}},"pull_request":{"url":"https://api.github.com/repos/CommBank/maestro/pulls/343","id":31890980,"html_url":"https://github.com/CommBank/maestro/pull/343","diff_url":"https://github.com/CommBank/maestro/pull/343.diff","patch_url":"https://github.com/CommBank/maestro/pull/343.patch","issue_url":"https://api.github.com/repos/CommBank/maestro/issues/343","number":343,"state":"open","locked":false,"title":"splitting uploads","user":{"login":"jonjuscorpion666","id":4725101,"avatar_url":"https://avatars.githubusercontent.com/u/4725101?v=3","gravatar_id":"","url":"https://api.github.com/users/jonjuscorpion666","html_url":"https://github.com/jonjuscorpion666","followers_url":"https://api.github.com/users/jonjuscorpion666/followers","following_url":"https://api.github.com/users/jonjuscorpion666/following{/other_user}","gists_url":"https://api.github.com/users/jonjuscorpion666/gists{/gist_id}","starred_url":"https://api.github.com/users/jonjuscorpion666/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jonjuscorpion666/subscriptions","organizations_url":"https://api.github.com/users/jonjuscorpion666/orgs","repos_url":"https://api.github.com/users/jonjuscorpion666/repos","events_url":"https://api.github.com/users/jonjuscorpion666/events{/privacy}","received_events_url":"https://api.github.com/users/jonjuscorpion666/received_events","type":"User","site_admin":false},"body":"Splitting of uploads into 2\r\n\r\n1) File Matching Module\r\n2) File Upload Module\r\n3)Backward compactability with current uploads","created_at":"2015-03-25T03:38:34Z","updated_at":"2015-03-30T05:08:59Z","closed_at":null,"merged_at":null,"merge_commit_sha":"4807f69f66af8a6e98273b650f12640e3363d44f","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/CommBank/maestro/pulls/343/commits","review_comments_url":"https://api.github.com/repos/CommBank/maestro/pulls/343/comments","review_comment_url":"https://api.github.com/repos/CommBank/maestro/pulls/comments{/number}","comments_url":"https://api.github.com/repos/CommBank/maestro/issues/343/comments","statuses_url":"https://api.github.com/repos/CommBank/maestro/statuses/ba41bf5603544ffa63483a9416f84dc346f9bf1c","head":{"label":"CommBank:HTValidatation","ref":"HTValidatation","sha":"ba41bf5603544ffa63483a9416f84dc346f9bf1c","user":{"login":"CommBank","id":4240076,"avatar_url":"https://avatars.githubusercontent.com/u/4240076?v=3","gravatar_id":"","url":"https://api.github.com/users/CommBank","html_url":"https://github.com/CommBank","followers_url":"https://api.github.com/users/CommBank/followers","following_url":"https://api.github.com/users/CommBank/following{/other_user}","gists_url":"https://api.github.com/users/CommBank/gists{/gist_id}","starred_url":"https://api.github.com/users/CommBank/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CommBank/subscriptions","organizations_url":"https://api.github.com/users/CommBank/orgs","repos_url":"https://api.github.com/users/CommBank/repos","events_url":"https://api.github.com/users/CommBank/events{/privacy}","received_events_url":"https://api.github.com/users/CommBank/received_events","type":"Organization","site_admin":false},"repo":{"id":17347463,"name":"maestro","full_name":"CommBank/maestro","owner":{"login":"CommBank","id":4240076,"avatar_url":"https://avatars.githubusercontent.com/u/4240076?v=3","gravatar_id":"","url":"https://api.github.com/users/CommBank","html_url":"https://github.com/CommBank","followers_url":"https://api.github.com/users/CommBank/followers","following_url":"https://api.github.com/users/CommBank/following{/other_user}","gists_url":"https://api.github.com/users/CommBank/gists{/gist_id}","starred_url":"https://api.github.com/users/CommBank/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CommBank/subscriptions","organizations_url":"https://api.github.com/users/CommBank/orgs","repos_url":"https://api.github.com/users/CommBank/repos","events_url":"https://api.github.com/users/CommBank/events{/privacy}","received_events_url":"https://api.github.com/users/CommBank/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/CommBank/maestro","description":"Data Pipelines that are well typed and enforce good convention","fork":false,"url":"https://api.github.com/repos/CommBank/maestro","forks_url":"https://api.github.com/repos/CommBank/maestro/forks","keys_url":"https://api.github.com/repos/CommBank/maestro/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CommBank/maestro/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CommBank/maestro/teams","hooks_url":"https://api.github.com/repos/CommBank/maestro/hooks","issue_events_url":"https://api.github.com/repos/CommBank/maestro/issues/events{/number}","events_url":"https://api.github.com/repos/CommBank/maestro/events","assignees_url":"https://api.github.com/repos/CommBank/maestro/assignees{/user}","branches_url":"https://api.github.com/repos/CommBank/maestro/branches{/branch}","tags_url":"https://api.github.com/repos/CommBank/maestro/tags","blobs_url":"https://api.github.com/repos/CommBank/maestro/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CommBank/maestro/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CommBank/maestro/git/refs{/sha}","trees_url":"https://api.github.com/repos/CommBank/maestro/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CommBank/maestro/statuses/{sha}","languages_url":"https://api.github.com/repos/CommBank/maestro/languages","stargazers_url":"https://api.github.com/repos/CommBank/maestro/stargazers","contributors_url":"https://api.github.com/repos/CommBank/maestro/contributors","subscribers_url":"https://api.github.com/repos/CommBank/maestro/subscribers","subscription_url":"https://api.github.com/repos/CommBank/maestro/subscription","commits_url":"https://api.github.com/repos/CommBank/maestro/commits{/sha}","git_commits_url":"https://api.github.com/repos/CommBank/maestro/git/commits{/sha}","comments_url":"https://api.github.com/repos/CommBank/maestro/comments{/number}","issue_comment_url":"https://api.github.com/repos/CommBank/maestro/issues/comments{/number}","contents_url":"https://api.github.com/repos/CommBank/maestro/contents/{+path}","compare_url":"https://api.github.com/repos/CommBank/maestro/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CommBank/maestro/merges","archive_url":"https://api.github.com/repos/CommBank/maestro/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CommBank/maestro/downloads","issues_url":"https://api.github.com/repos/CommBank/maestro/issues{/number}","pulls_url":"https://api.github.com/repos/CommBank/maestro/pulls{/number}","milestones_url":"https://api.github.com/repos/CommBank/maestro/milestones{/number}","notifications_url":"https://api.github.com/repos/CommBank/maestro/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CommBank/maestro/labels{/name}","releases_url":"https://api.github.com/repos/CommBank/maestro/releases{/id}","created_at":"2014-03-02T21:34:53Z","updated_at":"2015-03-26T06:39:40Z","pushed_at":"2015-03-29T21:56:20Z","git_url":"git://github.com/CommBank/maestro.git","ssh_url":"git@github.com:CommBank/maestro.git","clone_url":"https://github.com/CommBank/maestro.git","svn_url":"https://github.com/CommBank/maestro","homepage":"https://commbank.github.io/maestro","size":38933,"stargazers_count":27,"watchers_count":27,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":16,"mirror_url":null,"open_issues_count":65,"forks":16,"open_issues":65,"watchers":27,"default_branch":"master"}},"base":{"label":"CommBank:master","ref":"master","sha":"ab7d0e953552c17b2e6a6f0a17d1a651fcc62497","user":{"login":"CommBank","id":4240076,"avatar_url":"https://avatars.githubusercontent.com/u/4240076?v=3","gravatar_id":"","url":"https://api.github.com/users/CommBank","html_url":"https://github.com/CommBank","followers_url":"https://api.github.com/users/CommBank/followers","following_url":"https://api.github.com/users/CommBank/following{/other_user}","gists_url":"https://api.github.com/users/CommBank/gists{/gist_id}","starred_url":"https://api.github.com/users/CommBank/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CommBank/subscriptions","organizations_url":"https://api.github.com/users/CommBank/orgs","repos_url":"https://api.github.com/users/CommBank/repos","events_url":"https://api.github.com/users/CommBank/events{/privacy}","received_events_url":"https://api.github.com/users/CommBank/received_events","type":"Organization","site_admin":false},"repo":{"id":17347463,"name":"maestro","full_name":"CommBank/maestro","owner":{"login":"CommBank","id":4240076,"avatar_url":"https://avatars.githubusercontent.com/u/4240076?v=3","gravatar_id":"","url":"https://api.github.com/users/CommBank","html_url":"https://github.com/CommBank","followers_url":"https://api.github.com/users/CommBank/followers","following_url":"https://api.github.com/users/CommBank/following{/other_user}","gists_url":"https://api.github.com/users/CommBank/gists{/gist_id}","starred_url":"https://api.github.com/users/CommBank/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CommBank/subscriptions","organizations_url":"https://api.github.com/users/CommBank/orgs","repos_url":"https://api.github.com/users/CommBank/repos","events_url":"https://api.github.com/users/CommBank/events{/privacy}","received_events_url":"https://api.github.com/users/CommBank/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/CommBank/maestro","description":"Data Pipelines that are well typed and enforce good convention","fork":false,"url":"https://api.github.com/repos/CommBank/maestro","forks_url":"https://api.github.com/repos/CommBank/maestro/forks","keys_url":"https://api.github.com/repos/CommBank/maestro/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CommBank/maestro/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CommBank/maestro/teams","hooks_url":"https://api.github.com/repos/CommBank/maestro/hooks","issue_events_url":"https://api.github.com/repos/CommBank/maestro/issues/events{/number}","events_url":"https://api.github.com/repos/CommBank/maestro/events","assignees_url":"https://api.github.com/repos/CommBank/maestro/assignees{/user}","branches_url":"https://api.github.com/repos/CommBank/maestro/branches{/branch}","tags_url":"https://api.github.com/repos/CommBank/maestro/tags","blobs_url":"https://api.github.com/repos/CommBank/maestro/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CommBank/maestro/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CommBank/maestro/git/refs{/sha}","trees_url":"https://api.github.com/repos/CommBank/maestro/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CommBank/maestro/statuses/{sha}","languages_url":"https://api.github.com/repos/CommBank/maestro/languages","stargazers_url":"https://api.github.com/repos/CommBank/maestro/stargazers","contributors_url":"https://api.github.com/repos/CommBank/maestro/contributors","subscribers_url":"https://api.github.com/repos/CommBank/maestro/subscribers","subscription_url":"https://api.github.com/repos/CommBank/maestro/subscription","commits_url":"https://api.github.com/repos/CommBank/maestro/commits{/sha}","git_commits_url":"https://api.github.com/repos/CommBank/maestro/git/commits{/sha}","comments_url":"https://api.github.com/repos/CommBank/maestro/comments{/number}","issue_comment_url":"https://api.github.com/repos/CommBank/maestro/issues/comments{/number}","contents_url":"https://api.github.com/repos/CommBank/maestro/contents/{+path}","compare_url":"https://api.github.com/repos/CommBank/maestro/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CommBank/maestro/merges","archive_url":"https://api.github.com/repos/CommBank/maestro/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CommBank/maestro/downloads","issues_url":"https://api.github.com/repos/CommBank/maestro/issues{/number}","pulls_url":"https://api.github.com/repos/CommBank/maestro/pulls{/number}","milestones_url":"https://api.github.com/repos/CommBank/maestro/milestones{/number}","notifications_url":"https://api.github.com/repos/CommBank/maestro/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CommBank/maestro/labels{/name}","releases_url":"https://api.github.com/repos/CommBank/maestro/releases{/id}","created_at":"2014-03-02T21:34:53Z","updated_at":"2015-03-26T06:39:40Z","pushed_at":"2015-03-29T21:56:20Z","git_url":"git://github.com/CommBank/maestro.git","ssh_url":"git@github.com:CommBank/maestro.git","clone_url":"https://github.com/CommBank/maestro.git","svn_url":"https://github.com/CommBank/maestro","homepage":"https://commbank.github.io/maestro","size":38933,"stargazers_count":27,"watchers_count":27,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":16,"mirror_url":null,"open_issues_count":65,"forks":16,"open_issues":65,"watchers":27,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/CommBank/maestro/pulls/343"},"html":{"href":"https://github.com/CommBank/maestro/pull/343"},"issue":{"href":"https://api.github.com/repos/CommBank/maestro/issues/343"},"comments":{"href":"https://api.github.com/repos/CommBank/maestro/issues/343/comments"},"review_comments":{"href":"https://api.github.com/repos/CommBank/maestro/pulls/343/comments"},"review_comment":{"href":"https://api.github.com/repos/CommBank/maestro/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/CommBank/maestro/pulls/343/commits"},"statuses":{"href":"https://api.github.com/repos/CommBank/maestro/statuses/ba41bf5603544ffa63483a9416f84dc346f9bf1c"}}}}