{"action":"created","comment":{"url":"https://api.github.com/repos/codalab/codalab-cli/pulls/comments/46243475","id":46243475,"diff_hunk":"@@ -7,83 +7,95 @@\n file/directory.  In other words, zip files represent unnamed file/directories.\n \n To zip/unzip, we use the standard temp files.\n-'''\n+\"\"\"\n import os\n import shutil\n import sys\n+import subprocess\n import tempfile\n-from zipfile import ZipFile\n \n from codalab.common import UsageError\n-from codalab.lib import path_util, print_util\n+from codalab.lib import path_util, print_util, file_util\n \n-def zip(path, follow_symlinks, exclude_patterns, file_name):\n-    '''\n-    Take a path to a file or directory |path| and return the path to a zip archive\n-    containing its contents.  |file_name| is what the zip archive contains.\n-    '''\n-    if isinstance(path, list):\n-        for p in path:\n-            absolute_path = path_util.normalize(p)\n-            path_util.check_isvalid(absolute_path, 'zip_directory')\n-    else:\n-        absolute_path = path_util.normalize(path)\n-        path_util.check_isvalid(absolute_path, 'zip_directory')\n+# Files with these extensions are considered archive.\n+ARCHIVE_EXTS = ['.tar.gz', '.tgz', '.tar.bz2', '.zip']\n \n-    # Recursively copy the directory into a temp directory.\n-    temp_path = tempfile.mkdtemp()\n-    temp_subpath = os.path.join(temp_path, file_name)\n+# When deciding whether an archive contains a single file/directory, ignore\n+# these contents.\n+IGNORE_FILES = ['.DS_Store', '__MACOSX']\n \n-    print_util.open_line('Copying %s to %s' % (path, temp_subpath))\n-    if isinstance(path, list):\n-        os.mkdir(temp_subpath)\n-        for p in path:\n-            absolute_path = path_util.normalize(p)\n-            path_util.copy(absolute_path, os.path.join(temp_subpath, os.path.basename(p)), follow_symlinks=follow_symlinks, exclude_patterns=exclude_patterns)\n-    else:\n-        absolute_path = path_util.normalize(path)\n-        path_util.copy(absolute_path, temp_subpath, follow_symlinks=follow_symlinks, exclude_patterns=exclude_patterns)\n-    print_util.clear_line()\n+def path_is_archive(path):\n+    if isinstance(path, basestring):\n+        for ext in ARCHIVE_EXTS:\n+            if path.endswith(ext):\n+                return True\n+    return False\n \n-    zip_path = temp_path + '.zip'\n-    opts = '-qr'\n-    if not follow_symlinks: opts += ' --symlinks'\n-    print_util.open_line('Zipping to %s' % zip_path)\n-    if os.system(\"cd %s && zip %s %s %s\" % (temp_path, opts, zip_path, file_name)) != 0:\n-        raise UsageError('zip failed')\n \n-    path_util.remove(temp_path)\n-    return zip_path\n+def strip_archive_ext(path):\n+    for ext in ARCHIVE_EXTS:\n+        if path.endswith(ext):\n+            return path[:-len(ext)]\n+    raise UsageError('Not an archive: %s' % path)\n \n \n-def unzip(zip_path, temp_path, file_name):\n-    '''\n-    Take an absolute path to a zip file |zip_path| and return the path to a file or\n-    directory called |file_name| in |temp_path| containing its unzipped\n-    contents.\n-    Assume the zip file contains one file/directory called |file_name|.\n-    If |file_name| is not specified, then return the temp_path itself.\n-    '''\n-    path_util.check_isfile(zip_path, 'unzip_directory')\n-    if file_name:\n-        temp_subpath = os.path.join(temp_path, file_name)\n-    else:\n-        temp_subpath = temp_path\n+def add_packed_suffix(path):\n+    \"\"\"\n+    Add the packed suffix for path if it's not an archive.\n+    \"\"\"\n+    if path_is_archive(path):\n+        return path\n+    return path + '.tar.gz'\n+\n \n-    print_util.open_line('Unzipping %s to %s' % (zip_path, temp_subpath))\n-    if os.system(\"cd %s && unzip -q %s\" % (temp_path, zip_path)) != 0:\n-        raise UsageError('unzip failed')\n-    print_util.clear_line()\n-    # Corner case: note that the temp_subpath might not 'exist' because it is a\n-    # symlink (which is broken until it's put in the right place).\n-    if not os.path.exists(temp_subpath) and not os.path.islink(temp_subpath):\n-        raise UsageError('Zip file %s missing %s (%s doesn\\'t exist)' % (zip_path, file_name, temp_subpath))\n+def open_packed_path(source, follow_symlinks, exclude_patterns):\n+    \"\"\"\n+    Return file handle corresponding to |source|, which is either\n+    - an archive file: just stream it.\n+    - else: turn it into an archive\n+    \"\"\"\n+    if path_is_archive(source):\n+        return open(source)\n+    args = ['tar', 'cfz', '-', '-C', os.path.dirname(source) or '.', os.path.basename(source)]\n+    if follow_symlinks:\n+        args.append('-h')\n+    if exclude_patterns is not None:\n+        for pattern in exclude_patterns:\n+            args.append('--exclude=' + pattern)\n+    proc = subprocess.Popen(args, stdout=subprocess.PIPE)\n+    return proc.stdout\n \n-    return temp_subpath\n \n-def is_zip_file(path):\n-    try:\n-        ZipFile(path)\n-        return True\n-    except:\n-        return False\n+def unpack(source, dest_path):\n+    \"\"\"\n+    Unpack the archive |source_path| to |dest_path|.\n+    Note: |source| can be a file handle or a path.\n+    \"\"\"\n+    # Unpack to a temporary location.\n+    tmp_path = tempfile.mkdtemp('-zip_util.unpack')\n+    if isinstance(source, basestring):\n+        source_path = source\n+        if source_path.endswith('tar.gz') or source_path.endswith('tgz'):\n+            subprocess.call(['tar', 'xfz', source_path, '-C', tmp_path])","path":"codalab/lib/zip_util.py","position":143,"original_position":143,"commit_id":"f2ffaf0a94b1a3ec1b8da37528f63ac177c244d8","original_commit_id":"f2ffaf0a94b1a3ec1b8da37528f63ac177c244d8","user":{"login":"percyliang","id":435384,"avatar_url":"https://avatars.githubusercontent.com/u/435384?v=3","gravatar_id":"","url":"https://api.github.com/users/percyliang","html_url":"https://github.com/percyliang","followers_url":"https://api.github.com/users/percyliang/followers","following_url":"https://api.github.com/users/percyliang/following{/other_user}","gists_url":"https://api.github.com/users/percyliang/gists{/gist_id}","starred_url":"https://api.github.com/users/percyliang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/percyliang/subscriptions","organizations_url":"https://api.github.com/users/percyliang/orgs","repos_url":"https://api.github.com/users/percyliang/repos","events_url":"https://api.github.com/users/percyliang/events{/privacy}","received_events_url":"https://api.github.com/users/percyliang/received_events","type":"User","site_admin":false},"body":"good point - added","created_at":"2015-12-01T06:15:31Z","updated_at":"2015-12-01T06:15:31Z","html_url":"https://github.com/codalab/codalab-cli/pull/262#discussion_r46243475","pull_request_url":"https://api.github.com/repos/codalab/codalab-cli/pulls/262","_links":{"self":{"href":"https://api.github.com/repos/codalab/codalab-cli/pulls/comments/46243475"},"html":{"href":"https://github.com/codalab/codalab-cli/pull/262#discussion_r46243475"},"pull_request":{"href":"https://api.github.com/repos/codalab/codalab-cli/pulls/262"}}},"pull_request":{"url":"https://api.github.com/repos/codalab/codalab-cli/pulls/262","id":51865927,"html_url":"https://github.com/codalab/codalab-cli/pull/262","diff_url":"https://github.com/codalab/codalab-cli/pull/262.diff","patch_url":"https://github.com/codalab/codalab-cli/pull/262.patch","issue_url":"https://api.github.com/repos/codalab/codalab-cli/issues/262","number":262,"state":"open","locked":false,"title":"Overhaul of the uploading/downloading system.","user":{"login":"percyliang","id":435384,"avatar_url":"https://avatars.githubusercontent.com/u/435384?v=3","gravatar_id":"","url":"https://api.github.com/users/percyliang","html_url":"https://github.com/percyliang","followers_url":"https://api.github.com/users/percyliang/followers","following_url":"https://api.github.com/users/percyliang/following{/other_user}","gists_url":"https://api.github.com/users/percyliang/gists{/gist_id}","starred_url":"https://api.github.com/users/percyliang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/percyliang/subscriptions","organizations_url":"https://api.github.com/users/percyliang/orgs","repos_url":"https://api.github.com/users/percyliang/repos","events_url":"https://api.github.com/users/percyliang/events{/privacy}","received_events_url":"https://api.github.com/users/percyliang/received_events","type":"User","site_admin":false},"body":"Note: changes the BundleClient interface, so users need to git pull on the codalab-cli.\r\n- Switched over to tar.gz instead of zip to allow streaming.  This way, when we\r\n  download a large bundle, stuff starts showing up on the client immediately.\r\n- Minimize the amount of copying to temporary files and zipping before even\r\n  starting to copy.\r\n- Support `cl upload https://path-to-git-repo --git`\r\n- Support `cl upload https://path-to-archive`, where archive is one of\r\n  {.tar.gz, .tgz, .tar.bz2, .zip} and the archive gets automatically unpacked.\r\n- Created more comprehensive tests for uploading/downloading (e.g., symlinks,\r\n  funny characters, etc.)\r\n- Created tests that allow testing copying bundles across instances.\r\n\r\n@kashizui please review","created_at":"2015-11-26T02:13:49Z","updated_at":"2015-12-01T06:15:31Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b4dfdc0e12832e3413ac48d5566768fed868e21d","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/codalab/codalab-cli/pulls/262/commits","review_comments_url":"https://api.github.com/repos/codalab/codalab-cli/pulls/262/comments","review_comment_url":"https://api.github.com/repos/codalab/codalab-cli/pulls/comments{/number}","comments_url":"https://api.github.com/repos/codalab/codalab-cli/issues/262/comments","statuses_url":"https://api.github.com/repos/codalab/codalab-cli/statuses/f2ffaf0a94b1a3ec1b8da37528f63ac177c244d8","head":{"label":"codalab:feature/upload","ref":"feature/upload","sha":"f2ffaf0a94b1a3ec1b8da37528f63ac177c244d8","user":{"login":"codalab","id":4633552,"avatar_url":"https://avatars.githubusercontent.com/u/4633552?v=3","gravatar_id":"","url":"https://api.github.com/users/codalab","html_url":"https://github.com/codalab","followers_url":"https://api.github.com/users/codalab/followers","following_url":"https://api.github.com/users/codalab/following{/other_user}","gists_url":"https://api.github.com/users/codalab/gists{/gist_id}","starred_url":"https://api.github.com/users/codalab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/codalab/subscriptions","organizations_url":"https://api.github.com/users/codalab/orgs","repos_url":"https://api.github.com/users/codalab/repos","events_url":"https://api.github.com/users/codalab/events{/privacy}","received_events_url":"https://api.github.com/users/codalab/received_events","type":"User","site_admin":false},"repo":{"id":27352490,"name":"codalab-cli","full_name":"codalab/codalab-cli","owner":{"login":"codalab","id":4633552,"avatar_url":"https://avatars.githubusercontent.com/u/4633552?v=3","gravatar_id":"","url":"https://api.github.com/users/codalab","html_url":"https://github.com/codalab","followers_url":"https://api.github.com/users/codalab/followers","following_url":"https://api.github.com/users/codalab/following{/other_user}","gists_url":"https://api.github.com/users/codalab/gists{/gist_id}","starred_url":"https://api.github.com/users/codalab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/codalab/subscriptions","organizations_url":"https://api.github.com/users/codalab/orgs","repos_url":"https://api.github.com/users/codalab/repos","events_url":"https://api.github.com/users/codalab/events{/privacy}","received_events_url":"https://api.github.com/users/codalab/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/codalab/codalab-cli","description":"Bundle service and command-line interface for supporting worksheets in CodaLab.","fork":false,"url":"https://api.github.com/repos/codalab/codalab-cli","forks_url":"https://api.github.com/repos/codalab/codalab-cli/forks","keys_url":"https://api.github.com/repos/codalab/codalab-cli/keys{/key_id}","collaborators_url":"https://api.github.com/repos/codalab/codalab-cli/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/codalab/codalab-cli/teams","hooks_url":"https://api.github.com/repos/codalab/codalab-cli/hooks","issue_events_url":"https://api.github.com/repos/codalab/codalab-cli/issues/events{/number}","events_url":"https://api.github.com/repos/codalab/codalab-cli/events","assignees_url":"https://api.github.com/repos/codalab/codalab-cli/assignees{/user}","branches_url":"https://api.github.com/repos/codalab/codalab-cli/branches{/branch}","tags_url":"https://api.github.com/repos/codalab/codalab-cli/tags","blobs_url":"https://api.github.com/repos/codalab/codalab-cli/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/codalab/codalab-cli/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/codalab/codalab-cli/git/refs{/sha}","trees_url":"https://api.github.com/repos/codalab/codalab-cli/git/trees{/sha}","statuses_url":"https://api.github.com/repos/codalab/codalab-cli/statuses/{sha}","languages_url":"https://api.github.com/repos/codalab/codalab-cli/languages","stargazers_url":"https://api.github.com/repos/codalab/codalab-cli/stargazers","contributors_url":"https://api.github.com/repos/codalab/codalab-cli/contributors","subscribers_url":"https://api.github.com/repos/codalab/codalab-cli/subscribers","subscription_url":"https://api.github.com/repos/codalab/codalab-cli/subscription","commits_url":"https://api.github.com/repos/codalab/codalab-cli/commits{/sha}","git_commits_url":"https://api.github.com/repos/codalab/codalab-cli/git/commits{/sha}","comments_url":"https://api.github.com/repos/codalab/codalab-cli/comments{/number}","issue_comment_url":"https://api.github.com/repos/codalab/codalab-cli/issues/comments{/number}","contents_url":"https://api.github.com/repos/codalab/codalab-cli/contents/{+path}","compare_url":"https://api.github.com/repos/codalab/codalab-cli/compare/{base}...{head}","merges_url":"https://api.github.com/repos/codalab/codalab-cli/merges","archive_url":"https://api.github.com/repos/codalab/codalab-cli/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/codalab/codalab-cli/downloads","issues_url":"https://api.github.com/repos/codalab/codalab-cli/issues{/number}","pulls_url":"https://api.github.com/repos/codalab/codalab-cli/pulls{/number}","milestones_url":"https://api.github.com/repos/codalab/codalab-cli/milestones{/number}","notifications_url":"https://api.github.com/repos/codalab/codalab-cli/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/codalab/codalab-cli/labels{/name}","releases_url":"https://api.github.com/repos/codalab/codalab-cli/releases{/id}","created_at":"2014-11-30T22:33:18Z","updated_at":"2015-10-16T08:56:24Z","pushed_at":"2015-11-30T19:57:34Z","git_url":"git://github.com/codalab/codalab-cli.git","ssh_url":"git@github.com:codalab/codalab-cli.git","clone_url":"https://github.com/codalab/codalab-cli.git","svn_url":"https://github.com/codalab/codalab-cli","homepage":null,"size":1885,"stargazers_count":6,"watchers_count":6,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":8,"mirror_url":null,"open_issues_count":49,"forks":8,"open_issues":49,"watchers":6,"default_branch":"master"}},"base":{"label":"codalab:develop","ref":"develop","sha":"1f7bd01f6db7d6a2c29f1e52c387aa89dd06049a","user":{"login":"codalab","id":4633552,"avatar_url":"https://avatars.githubusercontent.com/u/4633552?v=3","gravatar_id":"","url":"https://api.github.com/users/codalab","html_url":"https://github.com/codalab","followers_url":"https://api.github.com/users/codalab/followers","following_url":"https://api.github.com/users/codalab/following{/other_user}","gists_url":"https://api.github.com/users/codalab/gists{/gist_id}","starred_url":"https://api.github.com/users/codalab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/codalab/subscriptions","organizations_url":"https://api.github.com/users/codalab/orgs","repos_url":"https://api.github.com/users/codalab/repos","events_url":"https://api.github.com/users/codalab/events{/privacy}","received_events_url":"https://api.github.com/users/codalab/received_events","type":"User","site_admin":false},"repo":{"id":27352490,"name":"codalab-cli","full_name":"codalab/codalab-cli","owner":{"login":"codalab","id":4633552,"avatar_url":"https://avatars.githubusercontent.com/u/4633552?v=3","gravatar_id":"","url":"https://api.github.com/users/codalab","html_url":"https://github.com/codalab","followers_url":"https://api.github.com/users/codalab/followers","following_url":"https://api.github.com/users/codalab/following{/other_user}","gists_url":"https://api.github.com/users/codalab/gists{/gist_id}","starred_url":"https://api.github.com/users/codalab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/codalab/subscriptions","organizations_url":"https://api.github.com/users/codalab/orgs","repos_url":"https://api.github.com/users/codalab/repos","events_url":"https://api.github.com/users/codalab/events{/privacy}","received_events_url":"https://api.github.com/users/codalab/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/codalab/codalab-cli","description":"Bundle service and command-line interface for supporting worksheets in CodaLab.","fork":false,"url":"https://api.github.com/repos/codalab/codalab-cli","forks_url":"https://api.github.com/repos/codalab/codalab-cli/forks","keys_url":"https://api.github.com/repos/codalab/codalab-cli/keys{/key_id}","collaborators_url":"https://api.github.com/repos/codalab/codalab-cli/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/codalab/codalab-cli/teams","hooks_url":"https://api.github.com/repos/codalab/codalab-cli/hooks","issue_events_url":"https://api.github.com/repos/codalab/codalab-cli/issues/events{/number}","events_url":"https://api.github.com/repos/codalab/codalab-cli/events","assignees_url":"https://api.github.com/repos/codalab/codalab-cli/assignees{/user}","branches_url":"https://api.github.com/repos/codalab/codalab-cli/branches{/branch}","tags_url":"https://api.github.com/repos/codalab/codalab-cli/tags","blobs_url":"https://api.github.com/repos/codalab/codalab-cli/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/codalab/codalab-cli/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/codalab/codalab-cli/git/refs{/sha}","trees_url":"https://api.github.com/repos/codalab/codalab-cli/git/trees{/sha}","statuses_url":"https://api.github.com/repos/codalab/codalab-cli/statuses/{sha}","languages_url":"https://api.github.com/repos/codalab/codalab-cli/languages","stargazers_url":"https://api.github.com/repos/codalab/codalab-cli/stargazers","contributors_url":"https://api.github.com/repos/codalab/codalab-cli/contributors","subscribers_url":"https://api.github.com/repos/codalab/codalab-cli/subscribers","subscription_url":"https://api.github.com/repos/codalab/codalab-cli/subscription","commits_url":"https://api.github.com/repos/codalab/codalab-cli/commits{/sha}","git_commits_url":"https://api.github.com/repos/codalab/codalab-cli/git/commits{/sha}","comments_url":"https://api.github.com/repos/codalab/codalab-cli/comments{/number}","issue_comment_url":"https://api.github.com/repos/codalab/codalab-cli/issues/comments{/number}","contents_url":"https://api.github.com/repos/codalab/codalab-cli/contents/{+path}","compare_url":"https://api.github.com/repos/codalab/codalab-cli/compare/{base}...{head}","merges_url":"https://api.github.com/repos/codalab/codalab-cli/merges","archive_url":"https://api.github.com/repos/codalab/codalab-cli/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/codalab/codalab-cli/downloads","issues_url":"https://api.github.com/repos/codalab/codalab-cli/issues{/number}","pulls_url":"https://api.github.com/repos/codalab/codalab-cli/pulls{/number}","milestones_url":"https://api.github.com/repos/codalab/codalab-cli/milestones{/number}","notifications_url":"https://api.github.com/repos/codalab/codalab-cli/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/codalab/codalab-cli/labels{/name}","releases_url":"https://api.github.com/repos/codalab/codalab-cli/releases{/id}","created_at":"2014-11-30T22:33:18Z","updated_at":"2015-10-16T08:56:24Z","pushed_at":"2015-11-30T19:57:34Z","git_url":"git://github.com/codalab/codalab-cli.git","ssh_url":"git@github.com:codalab/codalab-cli.git","clone_url":"https://github.com/codalab/codalab-cli.git","svn_url":"https://github.com/codalab/codalab-cli","homepage":null,"size":1885,"stargazers_count":6,"watchers_count":6,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":8,"mirror_url":null,"open_issues_count":49,"forks":8,"open_issues":49,"watchers":6,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/codalab/codalab-cli/pulls/262"},"html":{"href":"https://github.com/codalab/codalab-cli/pull/262"},"issue":{"href":"https://api.github.com/repos/codalab/codalab-cli/issues/262"},"comments":{"href":"https://api.github.com/repos/codalab/codalab-cli/issues/262/comments"},"review_comments":{"href":"https://api.github.com/repos/codalab/codalab-cli/pulls/262/comments"},"review_comment":{"href":"https://api.github.com/repos/codalab/codalab-cli/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/codalab/codalab-cli/pulls/262/commits"},"statuses":{"href":"https://api.github.com/repos/codalab/codalab-cli/statuses/f2ffaf0a94b1a3ec1b8da37528f63ac177c244d8"}}}}