{"action":"created","comment":{"url":"https://api.github.com/repos/lsst/pipe_tasks/pulls/comments/42702767","id":42702767,"diff_hunk":"@@ -605,3 +622,298 @@ def makeDataRefList(self, namespace):\n             )\n             self.refList.append(dataRef)\n \n+def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask):\n+    \"\"\"Function to count the number of pixels with a specific mask in a footprint\n+    \"\"\"\n+    bbox = footprint.getBBox()\n+    bbox.clip(mask.getBBox(afwImage.PARENT))\n+    fp = afwImage.MaskU(bbox)\n+    subMask = mask.Factory(mask, bbox, afwImage.PARENT)\n+    afwDet.setMaskFromFootprint(fp, footprint, bitmask)\n+    return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,\n+                             (subMask.getArray() & ignoreMask) == 0).sum()\n+\n+\n+class SafeClipAssembleCoaddConfig(AssembleCoaddConfig):\n+    clipDetection = pexConfig.ConfigurableField(target=SourceDetectionTask,\n+                                      doc=\"Detect sources on difference between unclipped and clipped coadd\")\n+    minClipFootOverlap = pexConfig.Field(\n+        doc = \"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped\",\n+        dtype = float,\n+        default = 0.65\n+    )\n+    minClipFootOverlapSingle = pexConfig.Field(\n+        doc = \"Minimum fractional overlap of clipped footprint with visit DETECTED to be \" \\\n+            \"clipped when only one visit overlaps\",\n+        dtype = float,\n+        default = 0.5\n+    )\n+    minClipFootOverlapDouble = pexConfig.Field(\n+        doc = \"Minimum fractional overlap of clipped footprints with visit DETECTED to be \" \\\n+            \"clipped when two visits overlap\",\n+        dtype = float,\n+        default = 0.45\n+    )\n+    maxClipFootOverlapDouble = pexConfig.Field(\n+        doc = \"Maximum fractional overlap of clipped footprints with visit DETECTED when \" \\\n+            \"considering two visits\",\n+        dtype = float,\n+        default = 0.15\n+    )\n+    minBigOverlap = pexConfig.Field(\n+        doc = \"Minimum number of pixels in footprint to use DETECTED mask from the single visits \" \\\n+            \"when labeling clipped footprints\",\n+        dtype = int,\n+        default = 100\n+    )\n+\n+    def setDefaults(self):\n+        pexConfig.Config.setDefaults(self)\n+        self.clipDetection.reEstimateBackground = False\n+        self.clipDetection.returnOriginalFootprints = False\n+        self.clipDetection.thresholdPolarity = \"both\"\n+        self.clipDetection.thresholdValue = 2\n+        self.clipDetection.nSigmaToGrow = 4\n+        self.clipDetection.minPixels = 4\n+        self.clipDetection.isotropicGrow = True\n+        self.clipDetection.thresholdType = \"pixel_stdev\"\n+        self.sigmaClip = 1.5\n+        self.clipIter = 3\n+\n+\n+class SafeClipAssembleCoaddTask(AssembleCoaddTask):\n+    \"\"\"Assemble a coadd, being careful to clip areas with potential artifacts\n+\n+    This is done in a number of steps:\n+        - identify potential regions that need to be clipped\n+        - find how much overlap there is between each potential region and the DETECTED mask for each visit\n+        - apply a cut based on the visit overlaps to determine the clipped regions\n+        - for big areas do not use the region as determined from the coadd, but rather all overlapping\n+          footprints from the visit DETECTED plane.  This helps in flagging low surface brightness regions.\n+        - assemble the coadd, ignoring the clipped regions.\n+\n+    The cuts used to determine the clipped regions were found empirically by examining a lot of HSC data.\n+    It is unlikely that they will generalize well to other data sets.  The current algorithm is limited to\n+    identifying artifacts that are found in only one or two visits.  A different threshold is applied\n+    depending on the number overlapping visits.  The current thresholds work well at removing a majority\n+    of artifacts.\n+\n+    \"\"\"\n+    ConfigClass = SafeClipAssembleCoaddConfig\n+    _DefaultName = \"safeClipAssembleCoadd\"\n+\n+    def __init__(self, *args, **kwargs):\n+        super(SafeClipAssembleCoaddTask, self).__init__(*args, **kwargs)\n+        schema = afwTable.SourceTable.makeMinimalSchema()\n+        self.makeSubtask(\"clipDetection\", schema=schema)\n+\n+    def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList):\n+        \"\"\"Assemble the coadd for a region\n+\n+        Identify clipped regions by detecting objects on the difference between unclipped and clipped coadd\n+        and then flag these regions on the individual vists so they are ignored in the coaddition process\n+\n+        @param skyInfo: Patch geometry information, from getSkyInfo\n+        @param tempExpRefList: List of data reference to tempExp\n+        @param imageScalerList: List of image scalers\n+        @param weightList: List of weights\n+        @param bgModelList: List of background models from background matching\n+        return coadd exposure\n+        \"\"\"\n+        exp = self.buildDifferenceImage(skyInfo, tempExpRefList, imageScalerList, weightList, bgModelList)\n+        mask = exp.getMaskedImage().getMask()\n+        mask.addMaskPlane(\"CLIPPED\")\n+\n+        result = self.detectClip(exp, tempExpRefList)\n+\n+        self.log.info('Found %d clipped objects' % len(result.clipFootprints))\n+\n+        # Go to individual visits for big footprints\n+        maskClipValue = mask.getPlaneBitMask(\"CLIPPED\")\n+        maskDetValue = mask.getPlaneBitMask(\"DETECTED\") | mask.getPlaneBitMask(\"DETECTED_NEGATIVE\")\n+        bigFootprints = self.detectClipBig(result.tempExpClipList, result.clipFootprints, result.clipIndices,\n+                                           maskClipValue, maskDetValue)\n+\n+        # Create mask of the current clipped footprints\n+        maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))\n+        afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)\n+\n+        maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))\n+        afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)\n+        maskClip |= maskClipBig\n+\n+        # Now combine again using mean, ignoring CLIPPED pixels\n+        badMaskPlanes = self.config.badMaskPlanes[:]\n+        badMaskPlanes.append(\"CLIPPED\")\n+        badPixelMask = afwImage.MaskU.getPlaneBitMask(badMaskPlanes)\n+        coaddExp = AssembleCoaddTask.assemble(self,skyInfo, tempExpRefList, imageScalerList, weightList,","path":"python/lsst/pipe/tasks/assembleCoadd.py","position":289,"original_position":289,"commit_id":"2105159a6bcb971d725a6368830070adf8adabfc","original_commit_id":"2105159a6bcb971d725a6368830070adf8adabfc","user":{"login":"jdswinbank","id":460854,"avatar_url":"https://avatars.githubusercontent.com/u/460854?v=3","gravatar_id":"","url":"https://api.github.com/users/jdswinbank","html_url":"https://github.com/jdswinbank","followers_url":"https://api.github.com/users/jdswinbank/followers","following_url":"https://api.github.com/users/jdswinbank/following{/other_user}","gists_url":"https://api.github.com/users/jdswinbank/gists{/gist_id}","starred_url":"https://api.github.com/users/jdswinbank/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jdswinbank/subscriptions","organizations_url":"https://api.github.com/users/jdswinbank/orgs","repos_url":"https://api.github.com/users/jdswinbank/repos","events_url":"https://api.github.com/users/jdswinbank/events{/privacy}","received_events_url":"https://api.github.com/users/jdswinbank/received_events","type":"User","site_admin":false},"body":"Needs a space after `self`.","created_at":"2015-10-22T01:16:42Z","updated_at":"2015-10-22T01:16:42Z","html_url":"https://github.com/lsst/pipe_tasks/pull/20#discussion_r42702767","pull_request_url":"https://api.github.com/repos/lsst/pipe_tasks/pulls/20","_links":{"self":{"href":"https://api.github.com/repos/lsst/pipe_tasks/pulls/comments/42702767"},"html":{"href":"https://github.com/lsst/pipe_tasks/pull/20#discussion_r42702767"},"pull_request":{"href":"https://api.github.com/repos/lsst/pipe_tasks/pulls/20"}}},"pull_request":{"url":"https://api.github.com/repos/lsst/pipe_tasks/pulls/20","id":46194178,"html_url":"https://github.com/lsst/pipe_tasks/pull/20","diff_url":"https://github.com/lsst/pipe_tasks/pull/20.diff","patch_url":"https://github.com/lsst/pipe_tasks/pull/20.patch","issue_url":"https://api.github.com/repos/lsst/pipe_tasks/issues/20","number":20,"state":"open","locked":false,"title":"Tickets/DM-2915: Port safe coadd clipping from HSC","user":{"login":"rearmstr","id":1485496,"avatar_url":"https://avatars.githubusercontent.com/u/1485496?v=3","gravatar_id":"","url":"https://api.github.com/users/rearmstr","html_url":"https://github.com/rearmstr","followers_url":"https://api.github.com/users/rearmstr/followers","following_url":"https://api.github.com/users/rearmstr/following{/other_user}","gists_url":"https://api.github.com/users/rearmstr/gists{/gist_id}","starred_url":"https://api.github.com/users/rearmstr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rearmstr/subscriptions","organizations_url":"https://api.github.com/users/rearmstr/orgs","repos_url":"https://api.github.com/users/rearmstr/repos","events_url":"https://api.github.com/users/rearmstr/events{/privacy}","received_events_url":"https://api.github.com/users/rearmstr/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-09-28T18:58:50Z","updated_at":"2015-10-22T01:16:42Z","closed_at":null,"merged_at":null,"merge_commit_sha":"51d951491e4ae089bc3290612de0810f2d3deaa7","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/lsst/pipe_tasks/pulls/20/commits","review_comments_url":"https://api.github.com/repos/lsst/pipe_tasks/pulls/20/comments","review_comment_url":"https://api.github.com/repos/lsst/pipe_tasks/pulls/comments{/number}","comments_url":"https://api.github.com/repos/lsst/pipe_tasks/issues/20/comments","statuses_url":"https://api.github.com/repos/lsst/pipe_tasks/statuses/2105159a6bcb971d725a6368830070adf8adabfc","head":{"label":"lsst:tickets/DM-2915","ref":"tickets/DM-2915","sha":"2105159a6bcb971d725a6368830070adf8adabfc","user":{"login":"lsst","id":1134504,"avatar_url":"https://avatars.githubusercontent.com/u/1134504?v=3","gravatar_id":"","url":"https://api.github.com/users/lsst","html_url":"https://github.com/lsst","followers_url":"https://api.github.com/users/lsst/followers","following_url":"https://api.github.com/users/lsst/following{/other_user}","gists_url":"https://api.github.com/users/lsst/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst/subscriptions","organizations_url":"https://api.github.com/users/lsst/orgs","repos_url":"https://api.github.com/users/lsst/repos","events_url":"https://api.github.com/users/lsst/events{/privacy}","received_events_url":"https://api.github.com/users/lsst/received_events","type":"Organization","site_admin":false},"repo":{"id":23013457,"name":"pipe_tasks","full_name":"lsst/pipe_tasks","owner":{"login":"lsst","id":1134504,"avatar_url":"https://avatars.githubusercontent.com/u/1134504?v=3","gravatar_id":"","url":"https://api.github.com/users/lsst","html_url":"https://github.com/lsst","followers_url":"https://api.github.com/users/lsst/followers","following_url":"https://api.github.com/users/lsst/following{/other_user}","gists_url":"https://api.github.com/users/lsst/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst/subscriptions","organizations_url":"https://api.github.com/users/lsst/orgs","repos_url":"https://api.github.com/users/lsst/repos","events_url":"https://api.github.com/users/lsst/events{/privacy}","received_events_url":"https://api.github.com/users/lsst/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/lsst/pipe_tasks","description":"LSST Data Management: astronomical data processing tasks","fork":false,"url":"https://api.github.com/repos/lsst/pipe_tasks","forks_url":"https://api.github.com/repos/lsst/pipe_tasks/forks","keys_url":"https://api.github.com/repos/lsst/pipe_tasks/keys{/key_id}","collaborators_url":"https://api.github.com/repos/lsst/pipe_tasks/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/lsst/pipe_tasks/teams","hooks_url":"https://api.github.com/repos/lsst/pipe_tasks/hooks","issue_events_url":"https://api.github.com/repos/lsst/pipe_tasks/issues/events{/number}","events_url":"https://api.github.com/repos/lsst/pipe_tasks/events","assignees_url":"https://api.github.com/repos/lsst/pipe_tasks/assignees{/user}","branches_url":"https://api.github.com/repos/lsst/pipe_tasks/branches{/branch}","tags_url":"https://api.github.com/repos/lsst/pipe_tasks/tags","blobs_url":"https://api.github.com/repos/lsst/pipe_tasks/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/lsst/pipe_tasks/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/lsst/pipe_tasks/git/refs{/sha}","trees_url":"https://api.github.com/repos/lsst/pipe_tasks/git/trees{/sha}","statuses_url":"https://api.github.com/repos/lsst/pipe_tasks/statuses/{sha}","languages_url":"https://api.github.com/repos/lsst/pipe_tasks/languages","stargazers_url":"https://api.github.com/repos/lsst/pipe_tasks/stargazers","contributors_url":"https://api.github.com/repos/lsst/pipe_tasks/contributors","subscribers_url":"https://api.github.com/repos/lsst/pipe_tasks/subscribers","subscription_url":"https://api.github.com/repos/lsst/pipe_tasks/subscription","commits_url":"https://api.github.com/repos/lsst/pipe_tasks/commits{/sha}","git_commits_url":"https://api.github.com/repos/lsst/pipe_tasks/git/commits{/sha}","comments_url":"https://api.github.com/repos/lsst/pipe_tasks/comments{/number}","issue_comment_url":"https://api.github.com/repos/lsst/pipe_tasks/issues/comments{/number}","contents_url":"https://api.github.com/repos/lsst/pipe_tasks/contents/{+path}","compare_url":"https://api.github.com/repos/lsst/pipe_tasks/compare/{base}...{head}","merges_url":"https://api.github.com/repos/lsst/pipe_tasks/merges","archive_url":"https://api.github.com/repos/lsst/pipe_tasks/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/lsst/pipe_tasks/downloads","issues_url":"https://api.github.com/repos/lsst/pipe_tasks/issues{/number}","pulls_url":"https://api.github.com/repos/lsst/pipe_tasks/pulls{/number}","milestones_url":"https://api.github.com/repos/lsst/pipe_tasks/milestones{/number}","notifications_url":"https://api.github.com/repos/lsst/pipe_tasks/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/lsst/pipe_tasks/labels{/name}","releases_url":"https://api.github.com/repos/lsst/pipe_tasks/releases{/id}","created_at":"2014-08-16T07:50:15Z","updated_at":"2015-04-29T02:12:38Z","pushed_at":"2015-10-21T23:21:57Z","git_url":"git://github.com/lsst/pipe_tasks.git","ssh_url":"git@github.com:lsst/pipe_tasks.git","clone_url":"https://github.com/lsst/pipe_tasks.git","svn_url":"https://github.com/lsst/pipe_tasks","homepage":"http://dm.lsst.org/","size":13064,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":4,"forks":1,"open_issues":4,"watchers":0,"default_branch":"master"}},"base":{"label":"lsst:master","ref":"master","sha":"808c8c66472ee65cafbe03a83ec9f917b74ebb4b","user":{"login":"lsst","id":1134504,"avatar_url":"https://avatars.githubusercontent.com/u/1134504?v=3","gravatar_id":"","url":"https://api.github.com/users/lsst","html_url":"https://github.com/lsst","followers_url":"https://api.github.com/users/lsst/followers","following_url":"https://api.github.com/users/lsst/following{/other_user}","gists_url":"https://api.github.com/users/lsst/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst/subscriptions","organizations_url":"https://api.github.com/users/lsst/orgs","repos_url":"https://api.github.com/users/lsst/repos","events_url":"https://api.github.com/users/lsst/events{/privacy}","received_events_url":"https://api.github.com/users/lsst/received_events","type":"Organization","site_admin":false},"repo":{"id":23013457,"name":"pipe_tasks","full_name":"lsst/pipe_tasks","owner":{"login":"lsst","id":1134504,"avatar_url":"https://avatars.githubusercontent.com/u/1134504?v=3","gravatar_id":"","url":"https://api.github.com/users/lsst","html_url":"https://github.com/lsst","followers_url":"https://api.github.com/users/lsst/followers","following_url":"https://api.github.com/users/lsst/following{/other_user}","gists_url":"https://api.github.com/users/lsst/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst/subscriptions","organizations_url":"https://api.github.com/users/lsst/orgs","repos_url":"https://api.github.com/users/lsst/repos","events_url":"https://api.github.com/users/lsst/events{/privacy}","received_events_url":"https://api.github.com/users/lsst/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/lsst/pipe_tasks","description":"LSST Data Management: astronomical data processing tasks","fork":false,"url":"https://api.github.com/repos/lsst/pipe_tasks","forks_url":"https://api.github.com/repos/lsst/pipe_tasks/forks","keys_url":"https://api.github.com/repos/lsst/pipe_tasks/keys{/key_id}","collaborators_url":"https://api.github.com/repos/lsst/pipe_tasks/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/lsst/pipe_tasks/teams","hooks_url":"https://api.github.com/repos/lsst/pipe_tasks/hooks","issue_events_url":"https://api.github.com/repos/lsst/pipe_tasks/issues/events{/number}","events_url":"https://api.github.com/repos/lsst/pipe_tasks/events","assignees_url":"https://api.github.com/repos/lsst/pipe_tasks/assignees{/user}","branches_url":"https://api.github.com/repos/lsst/pipe_tasks/branches{/branch}","tags_url":"https://api.github.com/repos/lsst/pipe_tasks/tags","blobs_url":"https://api.github.com/repos/lsst/pipe_tasks/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/lsst/pipe_tasks/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/lsst/pipe_tasks/git/refs{/sha}","trees_url":"https://api.github.com/repos/lsst/pipe_tasks/git/trees{/sha}","statuses_url":"https://api.github.com/repos/lsst/pipe_tasks/statuses/{sha}","languages_url":"https://api.github.com/repos/lsst/pipe_tasks/languages","stargazers_url":"https://api.github.com/repos/lsst/pipe_tasks/stargazers","contributors_url":"https://api.github.com/repos/lsst/pipe_tasks/contributors","subscribers_url":"https://api.github.com/repos/lsst/pipe_tasks/subscribers","subscription_url":"https://api.github.com/repos/lsst/pipe_tasks/subscription","commits_url":"https://api.github.com/repos/lsst/pipe_tasks/commits{/sha}","git_commits_url":"https://api.github.com/repos/lsst/pipe_tasks/git/commits{/sha}","comments_url":"https://api.github.com/repos/lsst/pipe_tasks/comments{/number}","issue_comment_url":"https://api.github.com/repos/lsst/pipe_tasks/issues/comments{/number}","contents_url":"https://api.github.com/repos/lsst/pipe_tasks/contents/{+path}","compare_url":"https://api.github.com/repos/lsst/pipe_tasks/compare/{base}...{head}","merges_url":"https://api.github.com/repos/lsst/pipe_tasks/merges","archive_url":"https://api.github.com/repos/lsst/pipe_tasks/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/lsst/pipe_tasks/downloads","issues_url":"https://api.github.com/repos/lsst/pipe_tasks/issues{/number}","pulls_url":"https://api.github.com/repos/lsst/pipe_tasks/pulls{/number}","milestones_url":"https://api.github.com/repos/lsst/pipe_tasks/milestones{/number}","notifications_url":"https://api.github.com/repos/lsst/pipe_tasks/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/lsst/pipe_tasks/labels{/name}","releases_url":"https://api.github.com/repos/lsst/pipe_tasks/releases{/id}","created_at":"2014-08-16T07:50:15Z","updated_at":"2015-04-29T02:12:38Z","pushed_at":"2015-10-21T23:21:57Z","git_url":"git://github.com/lsst/pipe_tasks.git","ssh_url":"git@github.com:lsst/pipe_tasks.git","clone_url":"https://github.com/lsst/pipe_tasks.git","svn_url":"https://github.com/lsst/pipe_tasks","homepage":"http://dm.lsst.org/","size":13064,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":4,"forks":1,"open_issues":4,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/lsst/pipe_tasks/pulls/20"},"html":{"href":"https://github.com/lsst/pipe_tasks/pull/20"},"issue":{"href":"https://api.github.com/repos/lsst/pipe_tasks/issues/20"},"comments":{"href":"https://api.github.com/repos/lsst/pipe_tasks/issues/20/comments"},"review_comments":{"href":"https://api.github.com/repos/lsst/pipe_tasks/pulls/20/comments"},"review_comment":{"href":"https://api.github.com/repos/lsst/pipe_tasks/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/lsst/pipe_tasks/pulls/20/commits"},"statuses":{"href":"https://api.github.com/repos/lsst/pipe_tasks/statuses/2105159a6bcb971d725a6368830070adf8adabfc"}}}}