{"action":"created","comment":{"url":"https://api.github.com/repos/SciLifeLab/pm/pulls/comments/23451708","id":23451708,"diff_hunk":"@@ -0,0 +1,96 @@\n+\"\"\" PM controllers\n+\"\"\"\n+import os\n+import re\n+import shutil\n+\n+from cement.core import controller\n+\n+from pm.controllers import BaseController\n+from pm.utils import filesystem, misc\n+\n+class StorageController(BaseController):\n+    \"\"\" Storage Controller\n+\n+    Entry point for all functionalities related to storage\n+    \"\"\"\n+    class Meta:\n+        label = 'storage'\n+        description = \"Entry point for all functionalities related to storage\"\n+        stacked_on = 'base'\n+        stacked_type = 'nested'\n+        arguments = [\n+            (['-r', '--run'], dict(type=str, help=\"Work with a specific run\")),\n+            (['-d', '--days'], dict(type=int, default=10, help=\"Days to consider a run \\\"old\\\"\"))\n+        ]\n+\n+    #######################\n+    # Storage subcommands #\n+    #######################\n+\n+    @controller.expose(help=\"Move old runs to nosync directory so they're not synced to the processing server\")\n+    def cleanup(self):\n+        for data_dir in self.app.config.get('storage', 'data_dirs'):\n+            with filesystem.chdir(data_dir):\n+                for run in [r for r in os.listdir(data_dir) if re.match(filesystem.RUN_RE, r)]:\n+                    if os.path.exists(os.path.join(run, 'RTAComplete.txt')):\n+                        self.app.log.info('Moving run {} to nosync directory'.format(os.path.basename(run)))\n+                        shutil.move(run, 'nosync')\n+\n+\n+    @controller.expose(help=\"Archive old runs to SWESTORE\")\n+    def archive_to_swestore(self):\n+        # If the run is specified in the command line, check that exists and archive\n+        if self.app.pargs.run:\n+            if re.match(filesystem.RUN_RE, os.path.basename(self.app.pargs.run)):\n+                if not os.path.exists(self.app.pargs.run):\n+                    self.app.log.error((\"Run {} not found. Please make sure to specify \"\n+                        \"the absolute path or relative path being in the correct directory.\".format(self.app.pargs.run)))\n+                else:\n+                    self._archive_run(self.pargs.run)\n+            else:\n+                self.app.log.error(\"The name {} doesn't look like an Illumina run\".format(os.path.basename(run)))\n+        # Otherwise find all runs in every data dir on the nosync partition\n+        else:\n+            self.app.log.info(\"Archiving old runs to SWESTORE\")\n+            for data_dir in self.app.config.get('storage', 'data_dirs'):\n+                to_send_dir = os.path.join(data_dir, 'nosync')\n+                self.app.log.info('Checking {} directory'.format(to_send_dir))\n+                with filesystem.chdir(to_send_dir):\n+                    for run in [r for r in os.listdir(to_send_dir) if re.match(filesystem.RUN_RE, r)]:\n+                        self._archive_run(run)\n+\n+    #############################################################\n+    # Class helper methods, not exposed as commands/subcommands #\n+    #############################################################\n+    def _archive_run(self, run):\n+        \"\"\" Archive a specific run to swestore\n+\n+        :param str run: Run directory\n+        \"\"\"\n+        def _send_to_swestore(f, dest, remove=True):\n+            \"\"\" Send file to swestore checking adler32 on destination and eventually\n+            removing the file from disk\n+\n+            :param str f: File to remove\n+            :param str dest: Destination directory in Swestore\n+            :param bool remove: If True, remove original file from source\n+            \"\"\"\n+            self.app.log.info(\"Sending {} to swestore\".format(f))\n+            misc.call_external_command('iput -K -P {file} {dest}'.format(file=f, dest=dest),\n+                    with_log_files=True)\n+            self.app.log.info('Run {} sent correctly and checksum was okay.'.format(f))\n+            if remove:\n+                self.app.log.info('Removing run'.format(f))\n+                os.remove(f)\n+\n+\n+        if run.endswith('bz2'):\n+            _send_to_swestore(run, self.app.config.get('storage', 'irods').get('irodsHome'))\n+        else:\n+            self.app.log.info(\"Compressing run {}\".format(run))\n+            # Compress with pbzip2\n+            misc.call_external_command('tar --use-compress-program=pbzip2 -cf {run}.tar.bz2 {run}'.format(run=run))\n+            self.app.log.info('Run {} successfully compressed! Removing from disk...'.format(run))\n+            shutil.rmtree(run)","path":"pm/controllers/storage.py","position":95,"original_position":95,"commit_id":"fb855396a05468bd3bab8dc2b32d1910f54e77ef","original_commit_id":"fb855396a05468bd3bab8dc2b32d1910f54e77ef","user":{"login":"senthil10","id":5311007,"avatar_url":"https://avatars.githubusercontent.com/u/5311007?v=3","gravatar_id":"","url":"https://api.github.com/users/senthil10","html_url":"https://github.com/senthil10","followers_url":"https://api.github.com/users/senthil10/followers","following_url":"https://api.github.com/users/senthil10/following{/other_user}","gists_url":"https://api.github.com/users/senthil10/gists{/gist_id}","starred_url":"https://api.github.com/users/senthil10/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/senthil10/subscriptions","organizations_url":"https://api.github.com/users/senthil10/orgs","repos_url":"https://api.github.com/users/senthil10/repos","events_url":"https://api.github.com/users/senthil10/events{/privacy}","received_events_url":"https://api.github.com/users/senthil10/received_events","type":"User","site_admin":false},"body":"It looks fully safe, but I am still feeling paranoid :joy:\r\nLets delete everything :smiling_imp:","created_at":"2015-01-23T14:20:25Z","updated_at":"2015-01-23T14:20:25Z","html_url":"https://github.com/SciLifeLab/pm/pull/34#discussion_r23451708","pull_request_url":"https://api.github.com/repos/SciLifeLab/pm/pulls/34","_links":{"self":{"href":"https://api.github.com/repos/SciLifeLab/pm/pulls/comments/23451708"},"html":{"href":"https://github.com/SciLifeLab/pm/pull/34#discussion_r23451708"},"pull_request":{"href":"https://api.github.com/repos/SciLifeLab/pm/pulls/34"}}},"pull_request":{"url":"https://api.github.com/repos/SciLifeLab/pm/pulls/34","id":27919116,"html_url":"https://github.com/SciLifeLab/pm/pull/34","diff_url":"https://github.com/SciLifeLab/pm/pull/34.diff","patch_url":"https://github.com/SciLifeLab/pm/pull/34.patch","issue_url":"https://api.github.com/repos/SciLifeLab/pm/issues/34","number":34,"state":"open","locked":false,"title":"pm storage cleanup and pm storage archive-to-swestore","user":{"login":"guillermo-carrasco","id":1574147,"avatar_url":"https://avatars.githubusercontent.com/u/1574147?v=3","gravatar_id":"","url":"https://api.github.com/users/guillermo-carrasco","html_url":"https://github.com/guillermo-carrasco","followers_url":"https://api.github.com/users/guillermo-carrasco/followers","following_url":"https://api.github.com/users/guillermo-carrasco/following{/other_user}","gists_url":"https://api.github.com/users/guillermo-carrasco/gists{/gist_id}","starred_url":"https://api.github.com/users/guillermo-carrasco/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/guillermo-carrasco/subscriptions","organizations_url":"https://api.github.com/users/guillermo-carrasco/orgs","repos_url":"https://api.github.com/users/guillermo-carrasco/repos","events_url":"https://api.github.com/users/guillermo-carrasco/events{/privacy}","received_events_url":"https://api.github.com/users/guillermo-carrasco/received_events","type":"User","site_admin":false},"body":"* `pm storage cleanup` will go through all `data_dirs` in the config file, check for finished runs in there (presence of `RTAComplete.txt`) and move them to `nosync`\r\n* `pm storage archive-to-swestore` will go through all `nosync` directories within all `data_dirs` in the config file, compress raw data, send it to swestore and remove it from the disk\r\n\r\nI am **not** double checking adler32 calculations because it is actually done, with md5sum, during the transfer to SWESTORE with the [-K option](http://wiki.irods.org/index.php/iput).","created_at":"2015-01-23T12:02:31Z","updated_at":"2015-01-23T14:20:25Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6f3620fac968d9d09982d4fd1571d50d3e737dfd","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/SciLifeLab/pm/pulls/34/commits","review_comments_url":"https://api.github.com/repos/SciLifeLab/pm/pulls/34/comments","review_comment_url":"https://api.github.com/repos/SciLifeLab/pm/pulls/comments/{number}","comments_url":"https://api.github.com/repos/SciLifeLab/pm/issues/34/comments","statuses_url":"https://api.github.com/repos/SciLifeLab/pm/statuses/fb855396a05468bd3bab8dc2b32d1910f54e77ef","head":{"label":"guillermo-carrasco:master","ref":"master","sha":"fb855396a05468bd3bab8dc2b32d1910f54e77ef","user":{"login":"guillermo-carrasco","id":1574147,"avatar_url":"https://avatars.githubusercontent.com/u/1574147?v=3","gravatar_id":"","url":"https://api.github.com/users/guillermo-carrasco","html_url":"https://github.com/guillermo-carrasco","followers_url":"https://api.github.com/users/guillermo-carrasco/followers","following_url":"https://api.github.com/users/guillermo-carrasco/following{/other_user}","gists_url":"https://api.github.com/users/guillermo-carrasco/gists{/gist_id}","starred_url":"https://api.github.com/users/guillermo-carrasco/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/guillermo-carrasco/subscriptions","organizations_url":"https://api.github.com/users/guillermo-carrasco/orgs","repos_url":"https://api.github.com/users/guillermo-carrasco/repos","events_url":"https://api.github.com/users/guillermo-carrasco/events{/privacy}","received_events_url":"https://api.github.com/users/guillermo-carrasco/received_events","type":"User","site_admin":false},"repo":{"id":24139252,"name":"pm","full_name":"guillermo-carrasco/pm","owner":{"login":"guillermo-carrasco","id":1574147,"avatar_url":"https://avatars.githubusercontent.com/u/1574147?v=3","gravatar_id":"","url":"https://api.github.com/users/guillermo-carrasco","html_url":"https://github.com/guillermo-carrasco","followers_url":"https://api.github.com/users/guillermo-carrasco/followers","following_url":"https://api.github.com/users/guillermo-carrasco/following{/other_user}","gists_url":"https://api.github.com/users/guillermo-carrasco/gists{/gist_id}","starred_url":"https://api.github.com/users/guillermo-carrasco/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/guillermo-carrasco/subscriptions","organizations_url":"https://api.github.com/users/guillermo-carrasco/orgs","repos_url":"https://api.github.com/users/guillermo-carrasco/repos","events_url":"https://api.github.com/users/guillermo-carrasco/events{/privacy}","received_events_url":"https://api.github.com/users/guillermo-carrasco/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/guillermo-carrasco/pm","description":"Project and Data management for NGI Sweden","fork":false,"url":"https://api.github.com/repos/guillermo-carrasco/pm","forks_url":"https://api.github.com/repos/guillermo-carrasco/pm/forks","keys_url":"https://api.github.com/repos/guillermo-carrasco/pm/keys{/key_id}","collaborators_url":"https://api.github.com/repos/guillermo-carrasco/pm/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/guillermo-carrasco/pm/teams","hooks_url":"https://api.github.com/repos/guillermo-carrasco/pm/hooks","issue_events_url":"https://api.github.com/repos/guillermo-carrasco/pm/issues/events{/number}","events_url":"https://api.github.com/repos/guillermo-carrasco/pm/events","assignees_url":"https://api.github.com/repos/guillermo-carrasco/pm/assignees{/user}","branches_url":"https://api.github.com/repos/guillermo-carrasco/pm/branches{/branch}","tags_url":"https://api.github.com/repos/guillermo-carrasco/pm/tags","blobs_url":"https://api.github.com/repos/guillermo-carrasco/pm/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/guillermo-carrasco/pm/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/guillermo-carrasco/pm/git/refs{/sha}","trees_url":"https://api.github.com/repos/guillermo-carrasco/pm/git/trees{/sha}","statuses_url":"https://api.github.com/repos/guillermo-carrasco/pm/statuses/{sha}","languages_url":"https://api.github.com/repos/guillermo-carrasco/pm/languages","stargazers_url":"https://api.github.com/repos/guillermo-carrasco/pm/stargazers","contributors_url":"https://api.github.com/repos/guillermo-carrasco/pm/contributors","subscribers_url":"https://api.github.com/repos/guillermo-carrasco/pm/subscribers","subscription_url":"https://api.github.com/repos/guillermo-carrasco/pm/subscription","commits_url":"https://api.github.com/repos/guillermo-carrasco/pm/commits{/sha}","git_commits_url":"https://api.github.com/repos/guillermo-carrasco/pm/git/commits{/sha}","comments_url":"https://api.github.com/repos/guillermo-carrasco/pm/comments{/number}","issue_comment_url":"https://api.github.com/repos/guillermo-carrasco/pm/issues/comments/{number}","contents_url":"https://api.github.com/repos/guillermo-carrasco/pm/contents/{+path}","compare_url":"https://api.github.com/repos/guillermo-carrasco/pm/compare/{base}...{head}","merges_url":"https://api.github.com/repos/guillermo-carrasco/pm/merges","archive_url":"https://api.github.com/repos/guillermo-carrasco/pm/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/guillermo-carrasco/pm/downloads","issues_url":"https://api.github.com/repos/guillermo-carrasco/pm/issues{/number}","pulls_url":"https://api.github.com/repos/guillermo-carrasco/pm/pulls{/number}","milestones_url":"https://api.github.com/repos/guillermo-carrasco/pm/milestones{/number}","notifications_url":"https://api.github.com/repos/guillermo-carrasco/pm/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/guillermo-carrasco/pm/labels{/name}","releases_url":"https://api.github.com/repos/guillermo-carrasco/pm/releases{/id}","created_at":"2014-09-17T09:55:45Z","updated_at":"2015-01-23T11:53:10Z","pushed_at":"2015-01-23T11:53:10Z","git_url":"git://github.com/guillermo-carrasco/pm.git","ssh_url":"git@github.com:guillermo-carrasco/pm.git","clone_url":"https://github.com/guillermo-carrasco/pm.git","svn_url":"https://github.com/guillermo-carrasco/pm","homepage":null,"size":711,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"open_issues_count":0,"forks":3,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"SciLifeLab:master","ref":"master","sha":"cc7b5abc4cafc933288eab0cf5389e7854e19801","user":{"login":"SciLifeLab","id":713611,"avatar_url":"https://avatars.githubusercontent.com/u/713611?v=3","gravatar_id":"","url":"https://api.github.com/users/SciLifeLab","html_url":"https://github.com/SciLifeLab","followers_url":"https://api.github.com/users/SciLifeLab/followers","following_url":"https://api.github.com/users/SciLifeLab/following{/other_user}","gists_url":"https://api.github.com/users/SciLifeLab/gists{/gist_id}","starred_url":"https://api.github.com/users/SciLifeLab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/SciLifeLab/subscriptions","organizations_url":"https://api.github.com/users/SciLifeLab/orgs","repos_url":"https://api.github.com/users/SciLifeLab/repos","events_url":"https://api.github.com/users/SciLifeLab/events{/privacy}","received_events_url":"https://api.github.com/users/SciLifeLab/received_events","type":"Organization","site_admin":false},"repo":{"id":25529822,"name":"pm","full_name":"SciLifeLab/pm","owner":{"login":"SciLifeLab","id":713611,"avatar_url":"https://avatars.githubusercontent.com/u/713611?v=3","gravatar_id":"","url":"https://api.github.com/users/SciLifeLab","html_url":"https://github.com/SciLifeLab","followers_url":"https://api.github.com/users/SciLifeLab/followers","following_url":"https://api.github.com/users/SciLifeLab/following{/other_user}","gists_url":"https://api.github.com/users/SciLifeLab/gists{/gist_id}","starred_url":"https://api.github.com/users/SciLifeLab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/SciLifeLab/subscriptions","organizations_url":"https://api.github.com/users/SciLifeLab/orgs","repos_url":"https://api.github.com/users/SciLifeLab/repos","events_url":"https://api.github.com/users/SciLifeLab/events{/privacy}","received_events_url":"https://api.github.com/users/SciLifeLab/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/SciLifeLab/pm","description":"Project and Data management for NGI Sweden","fork":true,"url":"https://api.github.com/repos/SciLifeLab/pm","forks_url":"https://api.github.com/repos/SciLifeLab/pm/forks","keys_url":"https://api.github.com/repos/SciLifeLab/pm/keys{/key_id}","collaborators_url":"https://api.github.com/repos/SciLifeLab/pm/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/SciLifeLab/pm/teams","hooks_url":"https://api.github.com/repos/SciLifeLab/pm/hooks","issue_events_url":"https://api.github.com/repos/SciLifeLab/pm/issues/events{/number}","events_url":"https://api.github.com/repos/SciLifeLab/pm/events","assignees_url":"https://api.github.com/repos/SciLifeLab/pm/assignees{/user}","branches_url":"https://api.github.com/repos/SciLifeLab/pm/branches{/branch}","tags_url":"https://api.github.com/repos/SciLifeLab/pm/tags","blobs_url":"https://api.github.com/repos/SciLifeLab/pm/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/SciLifeLab/pm/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/SciLifeLab/pm/git/refs{/sha}","trees_url":"https://api.github.com/repos/SciLifeLab/pm/git/trees{/sha}","statuses_url":"https://api.github.com/repos/SciLifeLab/pm/statuses/{sha}","languages_url":"https://api.github.com/repos/SciLifeLab/pm/languages","stargazers_url":"https://api.github.com/repos/SciLifeLab/pm/stargazers","contributors_url":"https://api.github.com/repos/SciLifeLab/pm/contributors","subscribers_url":"https://api.github.com/repos/SciLifeLab/pm/subscribers","subscription_url":"https://api.github.com/repos/SciLifeLab/pm/subscription","commits_url":"https://api.github.com/repos/SciLifeLab/pm/commits{/sha}","git_commits_url":"https://api.github.com/repos/SciLifeLab/pm/git/commits{/sha}","comments_url":"https://api.github.com/repos/SciLifeLab/pm/comments{/number}","issue_comment_url":"https://api.github.com/repos/SciLifeLab/pm/issues/comments/{number}","contents_url":"https://api.github.com/repos/SciLifeLab/pm/contents/{+path}","compare_url":"https://api.github.com/repos/SciLifeLab/pm/compare/{base}...{head}","merges_url":"https://api.github.com/repos/SciLifeLab/pm/merges","archive_url":"https://api.github.com/repos/SciLifeLab/pm/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/SciLifeLab/pm/downloads","issues_url":"https://api.github.com/repos/SciLifeLab/pm/issues{/number}","pulls_url":"https://api.github.com/repos/SciLifeLab/pm/pulls{/number}","milestones_url":"https://api.github.com/repos/SciLifeLab/pm/milestones{/number}","notifications_url":"https://api.github.com/repos/SciLifeLab/pm/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/SciLifeLab/pm/labels{/name}","releases_url":"https://api.github.com/repos/SciLifeLab/pm/releases{/id}","created_at":"2014-10-21T15:23:34Z","updated_at":"2014-12-08T13:55:25Z","pushed_at":"2014-12-08T13:55:25Z","git_url":"git://github.com/SciLifeLab/pm.git","ssh_url":"git@github.com:SciLifeLab/pm.git","clone_url":"https://github.com/SciLifeLab/pm.git","svn_url":"https://github.com/SciLifeLab/pm","homepage":null,"size":497,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":21,"forks":2,"open_issues":21,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/SciLifeLab/pm/pulls/34"},"html":{"href":"https://github.com/SciLifeLab/pm/pull/34"},"issue":{"href":"https://api.github.com/repos/SciLifeLab/pm/issues/34"},"comments":{"href":"https://api.github.com/repos/SciLifeLab/pm/issues/34/comments"},"review_comments":{"href":"https://api.github.com/repos/SciLifeLab/pm/pulls/34/comments"},"review_comment":{"href":"https://api.github.com/repos/SciLifeLab/pm/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/SciLifeLab/pm/pulls/34/commits"},"statuses":{"href":"https://api.github.com/repos/SciLifeLab/pm/statuses/fb855396a05468bd3bab8dc2b32d1910f54e77ef"}}}}