{"action":"created","comment":{"url":"https://api.github.com/repos/DIRACGrid/DIRAC/pulls/comments/44652626","id":44652626,"diff_hunk":"@@ -0,0 +1,220 @@\n+\"\"\"A set of tools for the remote pilot agent logging system\n+\"\"\"\n+\n+__RCSID__ = \"$Id$\"\n+\n+import time\n+import json\n+from uuid import uuid1\n+import sys\n+import os\n+\n+def createPilotLoggerConfigFile( filename = 'PilotLogger.cfg',\n+                                 host = '',\n+                                 port = '',\n+                                 queuePath = '',\n+                                 key_file  = '',\n+                                 cert_file = '',\n+                                 ca_certs = '',\n+                                 fileWithID = ''):\n+  \"\"\"Helper function that creates proper configuration file.\n+     The format is json encoded file with the following options included\n+  \"\"\"\n+  keys = [\n+      'host',\n+      'port',\n+      'queuePath',\n+      'key_file',\n+      'cert_file',\n+      'ca_certs',\n+      'fileWithID'\n+      ]\n+  values = [\n+      host,\n+      port,\n+      queuePath,\n+      key_file,\n+      cert_file,\n+      ca_certs,\n+      fileWithID\n+      ]\n+  config = dict( zip( keys, values ) )\n+  config = json.dumps(config)\n+  with open(filename, 'w') as myFile:\n+    myFile.write(config)\n+\n+def readPilotLoggerConfigFile ( filename ):\n+  \"\"\"Helper function that loads configuration file.\n+  Returns:\n+    dict:\n+  \"\"\"\n+  try:\n+    with open(filename, 'r') as myFile:\n+      config = myFile.read()\n+      config = json.loads(config)\n+      return config\n+  except (IOError, ValueError):\n+    return None\n+\n+def generateDict( pilotUUID, status, minorStatus, timestamp, source ):\n+  \"\"\"Helper function that returs a dictionnary based on the\n+     set of input values.\n+  Returns\n+    dict:\n+  \"\"\"\n+\n+  keys = [\n+      'pilotUUID',\n+      'status',\n+      'minorStatus',\n+      'timestamp',\n+      'source'\n+      ]\n+  values = [\n+      pilotUUID,\n+      status,\n+      minorStatus,\n+      timestamp,\n+      source\n+      ]\n+  return dict( zip( keys, values ) )\n+\n+def encodeMessage( content ):\n+  \"\"\"Method encodes the message in form of the serialized JSON string\n+     see https://docs.python.org/2/library/json.html#py-to-json-table\n+  Args:\n+    content(dict):\n+  Returns:\n+    str: in the JSON format.\n+  Raises:\n+    TypeError:if cannont encode json properly\n+  \"\"\"\n+  return json.dumps( content )\n+\n+def decodeMessage( msgJSON ):\n+  \"\"\"Decodes the message from the serialized JSON string\n+     See https://docs.python.org/2/library/json.html#py-to-json-table.\n+  Args:\n+    msgJSON(str):in the JSON format.\n+  Returns:\n+    str: decoded objecst.\n+  Raises:\n+    TypeError: if cannot decode JSON properly.\n+  \"\"\"\n+  return json.loads( msgJSON )\n+\n+def isMessageFormatCorrect( content ):\n+  \"\"\"Checks if input format is correct.\n+     Function checks if the input format is a dictionnary\n+     in the following format:\n+     0) content is a dictionary,\n+     1) it contains only those keys of basestring types:\n+     'pilotUUID', 'status', 'minorStatus', 'timestamp', 'source',\n+     2) it contains only values of basestring types.\n+  Args:\n+    content(dict): all values must be non-empty\n+  Returns:\n+    bool: True if message format is correct, False otherwise\n+  Example:\n+    {\"status\": \"DIRAC Installation\",\n+     \"timestamp\": \"1427121370.7\",\n+      \"minorStatus\": \"Uname = Linux localhost 3.10.64-85.cernvm.x86_64\",\n+      \"pilotUUID\": \"eda78924-d169-11e4-bfd2-0800275d1a0a\",\n+      \"source\": \"pilot\"\n+      }\n+  \"\"\"\n+  if not isinstance( content, dict ):\n+    return False\n+  refKeys = [\n+      'pilotUUID',\n+      'status',\n+      'minorStatus',\n+      'timestamp',\n+      'source'\n+      ]\n+  refKeys.sort()\n+  keys = content.keys()\n+  keys.sort()\n+  if not keys == refKeys:\n+    return False\n+  values = content.values()\n+  # if any value is not of basestring type\n+  if any( not isinstance( val, basestring ) for val in values ):\n+    return False\n+  #checking if not empty for all\n+  values = content.values()\n+  if any( not val for val in values ):","path":"WorkloadManagementSystem/PilotAgent/PilotLogger/PilotLoggerTools.py","position":146,"original_position":146,"commit_id":"72d75c62ef60c8fb639cdb912a4d09eb264cd543","original_commit_id":"72d75c62ef60c8fb639cdb912a4d09eb264cd543","user":{"login":"chaen","id":3728211,"avatar_url":"https://avatars.githubusercontent.com/u/3728211?v=3","gravatar_id":"","url":"https://api.github.com/users/chaen","html_url":"https://github.com/chaen","followers_url":"https://api.github.com/users/chaen/followers","following_url":"https://api.github.com/users/chaen/following{/other_user}","gists_url":"https://api.github.com/users/chaen/gists{/gist_id}","starred_url":"https://api.github.com/users/chaen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chaen/subscriptions","organizations_url":"https://api.github.com/users/chaen/orgs","repos_url":"https://api.github.com/users/chaen/repos","events_url":"https://api.github.com/users/chaen/events{/privacy}","received_events_url":"https://api.github.com/users/chaen/received_events","type":"User","site_admin":false},"body":"Your comment says \"checking if not empty for all\", so that means that they should not all be empty, right ? Because this check is True if at least one of the value is None, so it is not the same. ","created_at":"2015-11-12T12:47:06Z","updated_at":"2015-11-12T12:47:06Z","html_url":"https://github.com/DIRACGrid/DIRAC/pull/2703#discussion_r44652626","pull_request_url":"https://api.github.com/repos/DIRACGrid/DIRAC/pulls/2703","_links":{"self":{"href":"https://api.github.com/repos/DIRACGrid/DIRAC/pulls/comments/44652626"},"html":{"href":"https://github.com/DIRACGrid/DIRAC/pull/2703#discussion_r44652626"},"pull_request":{"href":"https://api.github.com/repos/DIRACGrid/DIRAC/pulls/2703"}}},"pull_request":{"url":"https://api.github.com/repos/DIRACGrid/DIRAC/pulls/2703","id":49601345,"html_url":"https://github.com/DIRACGrid/DIRAC/pull/2703","diff_url":"https://github.com/DIRACGrid/DIRAC/pull/2703.diff","patch_url":"https://github.com/DIRACGrid/DIRAC/pull/2703.patch","issue_url":"https://api.github.com/repos/DIRACGrid/DIRAC/issues/2703","number":2703,"state":"open","locked":false,"title":"[v6r16] PilotLoggers","user":{"login":"wkrzemien","id":11519095,"avatar_url":"https://avatars.githubusercontent.com/u/11519095?v=3","gravatar_id":"","url":"https://api.github.com/users/wkrzemien","html_url":"https://github.com/wkrzemien","followers_url":"https://api.github.com/users/wkrzemien/followers","following_url":"https://api.github.com/users/wkrzemien/following{/other_user}","gists_url":"https://api.github.com/users/wkrzemien/gists{/gist_id}","starred_url":"https://api.github.com/users/wkrzemien/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wkrzemien/subscriptions","organizations_url":"https://api.github.com/users/wkrzemien/orgs","repos_url":"https://api.github.com/users/wkrzemien/repos","events_url":"https://api.github.com/users/wkrzemien/events{/privacy}","received_events_url":"https://api.github.com/users/wkrzemien/received_events","type":"User","site_admin":false},"body":"This is a second part of the pilot loggins system:\r\nPilotLogger is used in the extended version (ExtendedLogger) of the Logger class, which is used by the pilot commands. The PilotLogger is by default off and the ExtendedLogger acts in exactly the same manner as the original Logger class.\r\nThe PilotLogger can send messages about the current status of the installation/configuration etc. to RabbitMQ server from which it will be retrieved by the consumer process and stored into Pilot DB\r\nThe messages contain timestamps, status information and the pilotUUID (not pilotID!) by which we can identify even pilots which dont have the pilotID assigned.\r\nIn addition, the PilotLogger is storing some of the messages locally just before checking the connection to the RabbitMQ server, if the connection is down, the messages are queued if the connection is back the whole set of messages are sent.\r\nThe current version uses STOMP protocol and ssl certificates for authentication\r\n********\r\nAdded getUniqueIdFromOS to set the uuid depending on the OS e.g. CREAM, GRID, VM et","created_at":"2015-11-03T19:39:31Z","updated_at":"2015-11-12T12:47:06Z","closed_at":null,"merged_at":null,"merge_commit_sha":"823a9e94d1a46acccc1920eb71cfb403af537a18","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/DIRACGrid/DIRAC/pulls/2703/commits","review_comments_url":"https://api.github.com/repos/DIRACGrid/DIRAC/pulls/2703/comments","review_comment_url":"https://api.github.com/repos/DIRACGrid/DIRAC/pulls/comments{/number}","comments_url":"https://api.github.com/repos/DIRACGrid/DIRAC/issues/2703/comments","statuses_url":"https://api.github.com/repos/DIRACGrid/DIRAC/statuses/72d75c62ef60c8fb639cdb912a4d09eb264cd543","head":{"label":"wkrzemien:pilot_logger_stomp_full_pilots","ref":"pilot_logger_stomp_full_pilots","sha":"72d75c62ef60c8fb639cdb912a4d09eb264cd543","user":{"login":"wkrzemien","id":11519095,"avatar_url":"https://avatars.githubusercontent.com/u/11519095?v=3","gravatar_id":"","url":"https://api.github.com/users/wkrzemien","html_url":"https://github.com/wkrzemien","followers_url":"https://api.github.com/users/wkrzemien/followers","following_url":"https://api.github.com/users/wkrzemien/following{/other_user}","gists_url":"https://api.github.com/users/wkrzemien/gists{/gist_id}","starred_url":"https://api.github.com/users/wkrzemien/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wkrzemien/subscriptions","organizations_url":"https://api.github.com/users/wkrzemien/orgs","repos_url":"https://api.github.com/users/wkrzemien/repos","events_url":"https://api.github.com/users/wkrzemien/events{/privacy}","received_events_url":"https://api.github.com/users/wkrzemien/received_events","type":"User","site_admin":false},"repo":{"id":32387566,"name":"DIRAC","full_name":"wkrzemien/DIRAC","owner":{"login":"wkrzemien","id":11519095,"avatar_url":"https://avatars.githubusercontent.com/u/11519095?v=3","gravatar_id":"","url":"https://api.github.com/users/wkrzemien","html_url":"https://github.com/wkrzemien","followers_url":"https://api.github.com/users/wkrzemien/followers","following_url":"https://api.github.com/users/wkrzemien/following{/other_user}","gists_url":"https://api.github.com/users/wkrzemien/gists{/gist_id}","starred_url":"https://api.github.com/users/wkrzemien/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wkrzemien/subscriptions","organizations_url":"https://api.github.com/users/wkrzemien/orgs","repos_url":"https://api.github.com/users/wkrzemien/repos","events_url":"https://api.github.com/users/wkrzemien/events{/privacy}","received_events_url":"https://api.github.com/users/wkrzemien/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/wkrzemien/DIRAC","description":"DIRAC Grid","fork":true,"url":"https://api.github.com/repos/wkrzemien/DIRAC","forks_url":"https://api.github.com/repos/wkrzemien/DIRAC/forks","keys_url":"https://api.github.com/repos/wkrzemien/DIRAC/keys{/key_id}","collaborators_url":"https://api.github.com/repos/wkrzemien/DIRAC/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/wkrzemien/DIRAC/teams","hooks_url":"https://api.github.com/repos/wkrzemien/DIRAC/hooks","issue_events_url":"https://api.github.com/repos/wkrzemien/DIRAC/issues/events{/number}","events_url":"https://api.github.com/repos/wkrzemien/DIRAC/events","assignees_url":"https://api.github.com/repos/wkrzemien/DIRAC/assignees{/user}","branches_url":"https://api.github.com/repos/wkrzemien/DIRAC/branches{/branch}","tags_url":"https://api.github.com/repos/wkrzemien/DIRAC/tags","blobs_url":"https://api.github.com/repos/wkrzemien/DIRAC/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/wkrzemien/DIRAC/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/wkrzemien/DIRAC/git/refs{/sha}","trees_url":"https://api.github.com/repos/wkrzemien/DIRAC/git/trees{/sha}","statuses_url":"https://api.github.com/repos/wkrzemien/DIRAC/statuses/{sha}","languages_url":"https://api.github.com/repos/wkrzemien/DIRAC/languages","stargazers_url":"https://api.github.com/repos/wkrzemien/DIRAC/stargazers","contributors_url":"https://api.github.com/repos/wkrzemien/DIRAC/contributors","subscribers_url":"https://api.github.com/repos/wkrzemien/DIRAC/subscribers","subscription_url":"https://api.github.com/repos/wkrzemien/DIRAC/subscription","commits_url":"https://api.github.com/repos/wkrzemien/DIRAC/commits{/sha}","git_commits_url":"https://api.github.com/repos/wkrzemien/DIRAC/git/commits{/sha}","comments_url":"https://api.github.com/repos/wkrzemien/DIRAC/comments{/number}","issue_comment_url":"https://api.github.com/repos/wkrzemien/DIRAC/issues/comments{/number}","contents_url":"https://api.github.com/repos/wkrzemien/DIRAC/contents/{+path}","compare_url":"https://api.github.com/repos/wkrzemien/DIRAC/compare/{base}...{head}","merges_url":"https://api.github.com/repos/wkrzemien/DIRAC/merges","archive_url":"https://api.github.com/repos/wkrzemien/DIRAC/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/wkrzemien/DIRAC/downloads","issues_url":"https://api.github.com/repos/wkrzemien/DIRAC/issues{/number}","pulls_url":"https://api.github.com/repos/wkrzemien/DIRAC/pulls{/number}","milestones_url":"https://api.github.com/repos/wkrzemien/DIRAC/milestones{/number}","notifications_url":"https://api.github.com/repos/wkrzemien/DIRAC/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/wkrzemien/DIRAC/labels{/name}","releases_url":"https://api.github.com/repos/wkrzemien/DIRAC/releases{/id}","created_at":"2015-03-17T10:43:42Z","updated_at":"2015-03-17T10:43:46Z","pushed_at":"2015-11-03T19:32:19Z","git_url":"git://github.com/wkrzemien/DIRAC.git","ssh_url":"git@github.com:wkrzemien/DIRAC.git","clone_url":"https://github.com/wkrzemien/DIRAC.git","svn_url":"https://github.com/wkrzemien/DIRAC","homepage":"http://diracgrid.org","size":27885,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"integration"}},"base":{"label":"DIRACGrid:rel-v6r15","ref":"rel-v6r15","sha":"9e71f1878a9d22c716b89ee742108a1d81a18a4f","user":{"login":"DIRACGrid","id":694842,"avatar_url":"https://avatars.githubusercontent.com/u/694842?v=3","gravatar_id":"","url":"https://api.github.com/users/DIRACGrid","html_url":"https://github.com/DIRACGrid","followers_url":"https://api.github.com/users/DIRACGrid/followers","following_url":"https://api.github.com/users/DIRACGrid/following{/other_user}","gists_url":"https://api.github.com/users/DIRACGrid/gists{/gist_id}","starred_url":"https://api.github.com/users/DIRACGrid/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DIRACGrid/subscriptions","organizations_url":"https://api.github.com/users/DIRACGrid/orgs","repos_url":"https://api.github.com/users/DIRACGrid/repos","events_url":"https://api.github.com/users/DIRACGrid/events{/privacy}","received_events_url":"https://api.github.com/users/DIRACGrid/received_events","type":"Organization","site_admin":false},"repo":{"id":1576831,"name":"DIRAC","full_name":"DIRACGrid/DIRAC","owner":{"login":"DIRACGrid","id":694842,"avatar_url":"https://avatars.githubusercontent.com/u/694842?v=3","gravatar_id":"","url":"https://api.github.com/users/DIRACGrid","html_url":"https://github.com/DIRACGrid","followers_url":"https://api.github.com/users/DIRACGrid/followers","following_url":"https://api.github.com/users/DIRACGrid/following{/other_user}","gists_url":"https://api.github.com/users/DIRACGrid/gists{/gist_id}","starred_url":"https://api.github.com/users/DIRACGrid/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DIRACGrid/subscriptions","organizations_url":"https://api.github.com/users/DIRACGrid/orgs","repos_url":"https://api.github.com/users/DIRACGrid/repos","events_url":"https://api.github.com/users/DIRACGrid/events{/privacy}","received_events_url":"https://api.github.com/users/DIRACGrid/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/DIRACGrid/DIRAC","description":"DIRAC Grid","fork":false,"url":"https://api.github.com/repos/DIRACGrid/DIRAC","forks_url":"https://api.github.com/repos/DIRACGrid/DIRAC/forks","keys_url":"https://api.github.com/repos/DIRACGrid/DIRAC/keys{/key_id}","collaborators_url":"https://api.github.com/repos/DIRACGrid/DIRAC/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/DIRACGrid/DIRAC/teams","hooks_url":"https://api.github.com/repos/DIRACGrid/DIRAC/hooks","issue_events_url":"https://api.github.com/repos/DIRACGrid/DIRAC/issues/events{/number}","events_url":"https://api.github.com/repos/DIRACGrid/DIRAC/events","assignees_url":"https://api.github.com/repos/DIRACGrid/DIRAC/assignees{/user}","branches_url":"https://api.github.com/repos/DIRACGrid/DIRAC/branches{/branch}","tags_url":"https://api.github.com/repos/DIRACGrid/DIRAC/tags","blobs_url":"https://api.github.com/repos/DIRACGrid/DIRAC/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/DIRACGrid/DIRAC/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/DIRACGrid/DIRAC/git/refs{/sha}","trees_url":"https://api.github.com/repos/DIRACGrid/DIRAC/git/trees{/sha}","statuses_url":"https://api.github.com/repos/DIRACGrid/DIRAC/statuses/{sha}","languages_url":"https://api.github.com/repos/DIRACGrid/DIRAC/languages","stargazers_url":"https://api.github.com/repos/DIRACGrid/DIRAC/stargazers","contributors_url":"https://api.github.com/repos/DIRACGrid/DIRAC/contributors","subscribers_url":"https://api.github.com/repos/DIRACGrid/DIRAC/subscribers","subscription_url":"https://api.github.com/repos/DIRACGrid/DIRAC/subscription","commits_url":"https://api.github.com/repos/DIRACGrid/DIRAC/commits{/sha}","git_commits_url":"https://api.github.com/repos/DIRACGrid/DIRAC/git/commits{/sha}","comments_url":"https://api.github.com/repos/DIRACGrid/DIRAC/comments{/number}","issue_comment_url":"https://api.github.com/repos/DIRACGrid/DIRAC/issues/comments{/number}","contents_url":"https://api.github.com/repos/DIRACGrid/DIRAC/contents/{+path}","compare_url":"https://api.github.com/repos/DIRACGrid/DIRAC/compare/{base}...{head}","merges_url":"https://api.github.com/repos/DIRACGrid/DIRAC/merges","archive_url":"https://api.github.com/repos/DIRACGrid/DIRAC/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/DIRACGrid/DIRAC/downloads","issues_url":"https://api.github.com/repos/DIRACGrid/DIRAC/issues{/number}","pulls_url":"https://api.github.com/repos/DIRACGrid/DIRAC/pulls{/number}","milestones_url":"https://api.github.com/repos/DIRACGrid/DIRAC/milestones{/number}","notifications_url":"https://api.github.com/repos/DIRACGrid/DIRAC/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/DIRACGrid/DIRAC/labels{/name}","releases_url":"https://api.github.com/repos/DIRACGrid/DIRAC/releases{/id}","created_at":"2011-04-06T11:47:53Z","updated_at":"2015-11-02T08:25:43Z","pushed_at":"2015-11-12T12:16:46Z","git_url":"git://github.com/DIRACGrid/DIRAC.git","ssh_url":"git@github.com:DIRACGrid/DIRAC.git","clone_url":"https://github.com/DIRACGrid/DIRAC.git","svn_url":"https://github.com/DIRACGrid/DIRAC","homepage":"http://diracgrid.org","size":66902,"stargazers_count":45,"watchers_count":45,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":88,"mirror_url":null,"open_issues_count":129,"forks":88,"open_issues":129,"watchers":45,"default_branch":"integration"}},"_links":{"self":{"href":"https://api.github.com/repos/DIRACGrid/DIRAC/pulls/2703"},"html":{"href":"https://github.com/DIRACGrid/DIRAC/pull/2703"},"issue":{"href":"https://api.github.com/repos/DIRACGrid/DIRAC/issues/2703"},"comments":{"href":"https://api.github.com/repos/DIRACGrid/DIRAC/issues/2703/comments"},"review_comments":{"href":"https://api.github.com/repos/DIRACGrid/DIRAC/pulls/2703/comments"},"review_comment":{"href":"https://api.github.com/repos/DIRACGrid/DIRAC/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/DIRACGrid/DIRAC/pulls/2703/commits"},"statuses":{"href":"https://api.github.com/repos/DIRACGrid/DIRAC/statuses/72d75c62ef60c8fb639cdb912a4d09eb264cd543"}}}}