{"action":"closed","issue":{"url":"https://api.github.com/repos/sebastianbarfort/sds/issues/22","labels_url":"https://api.github.com/repos/sebastianbarfort/sds/issues/22/labels{/name}","comments_url":"https://api.github.com/repos/sebastianbarfort/sds/issues/22/comments","events_url":"https://api.github.com/repos/sebastianbarfort/sds/issues/22/events","html_url":"https://github.com/sebastianbarfort/sds/issues/22","id":111374247,"number":22,"title":"Group 8: Assignment 1","user":{"login":"sundgaard","id":14147545,"avatar_url":"https://avatars.githubusercontent.com/u/14147545?v=3","gravatar_id":"","url":"https://api.github.com/users/sundgaard","html_url":"https://github.com/sundgaard","followers_url":"https://api.github.com/users/sundgaard/followers","following_url":"https://api.github.com/users/sundgaard/following{/other_user}","gists_url":"https://api.github.com/users/sundgaard/gists{/gist_id}","starred_url":"https://api.github.com/users/sundgaard/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sundgaard/subscriptions","organizations_url":"https://api.github.com/users/sundgaard/orgs","repos_url":"https://api.github.com/users/sundgaard/repos","events_url":"https://api.github.com/users/sundgaard/events{/privacy}","received_events_url":"https://api.github.com/users/sundgaard/received_events","type":"User","site_admin":false},"labels":[],"state":"closed","locked":false,"assignee":null,"milestone":null,"comments":1,"created_at":"2015-10-14T10:57:13Z","updated_at":"2015-10-25T18:46:31Z","closed_at":"2015-10-25T18:46:31Z","body":"####### ASSIGNMENT 1: DATA MANIPULATION AND VISUALIZATION\r\n#Group 8\r\n#Hand-in, October 8 2015\r\n\r\n# Group Members:\r\n#   Dennis Hansen\r\n#   Susanne Sundgaard Hansen\r\n#   Oskar Harmsen\r\n#   Ann-Sofie Hansen\r\n\r\n#Fix international characters (only works on MAC)\r\nSys.setlocale(\"LC_ALL\", \"pt_PT.UTF-8\")\r\n\r\n#Load packages\r\nlibrary(\"readr\")\r\nlibrary(\"lubridate\")\r\nlibrary(\"dplyr\")\r\nlibrary(\"ggplot2\")\r\nlibrary(\"XML\")\r\nlibrary(\"httr\")\r\nlibrary(\"stringr\")\r\n\r\n#Load file\r\ndf = tbl_df(read.csv(\"https://raw.githubusercontent.com/MuseumofModernArt/collection/master/Artworks.csv\"))\r\n\r\n\r\n### Question 1-3: Create and plot a dataframe with stock of paintings for each month in the year\r\n  \r\n  #Data cleaning\r\n\r\n    #Select relevant columns and remove observations without dates\r\n    df1 <- df %>%\r\n      select(DateAcquired, CuratorApproved, Department, Title)%>%\r\n      filter(!is.na(DateAcquired) & DateAcquired!=\"\")%>%\r\n      arrange(DateAcquired)\r\n    \r\n    #Correct a wrong entry in the data and rearrange\r\n    df1$DateAcquired[1:5] <- \"2009-11-17\"     #Note: the wrong entries show out on top\r\n    df1 <- df1 %>% arrange(DateAcquired)      #Rearrange data\r\n    \r\n    #Mark class as dates\r\n    df1$DateAcquired <- as.Date(df1$DateAcquired) # Mark as date\r\n    df1 <- df1 %>% filter(!is.na(DateAcquired)) #Remove observations that are now NA (one is wrongly entered as \"1941\")\r\n\r\n  #Creating the data frame\r\n  df2 <-  df1 %>%\r\n        #Map every observation to the ending date of that month\r\n        mutate(month = ceiling_date(DateAcquired, unit = \"month\")) %>%\r\n        group_by(as.factor(as.character(month)), CuratorApproved) %>%\r\n        #Create new dataframe that counts all NEW acquisitions by month and curator status \r\n        summarise(count.month = n())%>%                                 \r\n        ungroup()%>%\r\n        # Add column that cumulates the total stock across time\r\n        mutate(cum.count = cumsum(count.month))%>%\r\n        # Add column that cumulates by curator status\r\n        group_by(CuratorApproved)%>%\r\n        mutate(cum.count.cur = cumsum(count.month))\r\n\r\n  #Rename dataframe and set as class date\r\n    colnames(df2)[1] <- \"month\"\r\n    df2$month <- as.Date(df2$month)\r\n\r\n\r\n  #Make a plot of the total stock\r\n  #Base line plot of stock through time\r\n    p <-  ggplot(df2, aes(x=month, y=cum.count))+\r\n      geom_line(aes(color=\"red\"))+\r\n      scale_x_date(breaks=\"10 year\")+\r\n      ggtitle(\"Cumulative Acquistions, MoMA, by date\")+\r\n      xlab(\"Date\")+\r\n      ylab(\"Total number of artworks acquired, to date\")+\r\n      theme(plot.title = element_text(lineheight=.8, face=\"bold\"))+\r\n      guides(color=FALSE)\r\n    p\r\n\r\n  #Base line plot of stock through time, by Curator Approval\r\n    p <-  ggplot(df2, aes(x=month, y=cum.count.cur, group=CuratorApproved))+\r\n      geom_line(aes(color=CuratorApproved))+\r\n      scale_x_date(breaks=\"10 year\")+\r\n      ggtitle(\"Cumulative Acquistions, MoMA, by date and Curator Approval\")+\r\n      xlab(\"Date\")+\r\n      ylab(\"Total number of artworks acquired, to date\")+\r\n      theme(plot.title = element_text(lineheight=.8, face=\"bold\"))+\r\n      guides(color=guide_legend(reverse=TRUE, title=\"Approved \\nby curator\"))\r\n    p\r\n    \r\n### Question 4-5: Plot by department\r\n    #Preparing dataframe (syntax and approach similar to question 1-3)\r\n    df3 <-  df1 %>%\r\n      mutate(month = floor_date(DateAcquired, unit = \"month\")) %>%\r\n      group_by(as.factor(as.character(month)), Department)%>%\r\n      summarise(count.month = n())%>%\r\n      ungroup()%>%\r\n      mutate(cum.count = cumsum(count.month))%>%\r\n      group_by(Department)%>%\r\n      mutate(cum.count.dep = cumsum(count.month))\r\n    \r\n    colnames(df3)[1] <- \"month\"\r\n    df3$month <- as.Date(df3$month)\r\n\r\n    #Plot line graph by department\r\n    p <-  ggplot(df3, aes(x=month, y=cum.count.dep, group=Department))+\r\n      geom_line(aes(color=Department))+\r\n      scale_x_date(breaks = \"10 year\")+\r\n      ggtitle(\"Cumulative Acquistions, MoMA, by department and date\")+\r\n      xlab(\"Date\")+\r\n      ylab(\"Total number of artworks acquired, to date\")+\r\n      theme(plot.title = element_text(lineheight=.8, face=\"bold\"), legend.position=\"bottom\")\r\n    p\r\n    \r\n    print(\"'Prints and Illustrated books' has had the largest increase.\")\r\n    \r\n    \r\n### Question 6: List top 10 artists by number of paintings \r\n  \r\n    #Summarize works by artist\r\n    df6 <-  df %>%\r\n            filter(Artist!=\"\") %>% #Remove rows without specified artist\r\n            group_by(Artist) %>% #Group by artist\r\n            summarise(n_works = n()) %>% #Count works by each artist\r\n            ungroup()%>%\r\n            arrange(desc((n_works))) #Arrange by most works\r\n    df6[1:10,]  #List top 10\r\n    \r\n    \r\n### 7: List birth place of painters and color world map accordingly\r\n    \r\n    #Data cleaning\r\n    \r\n      #Select relevant columns and remove observations without dates\r\n      df7 <-  df %>%\r\n        select(Artist, ArtistBio) %>% #Select relevant variables\r\n        filter(Artist!=\"\" & ArtistBio!=\"\") %>% #Remove rows without specified artist\r\n        #Filter to obtain only one row per artist\r\n        group_by(Artist) %>% #Group by artist\r\n        mutate( rank = 1:n() ) %>%  #Create a vector counting works by each artist  \r\n        filter( rank == max(rank) ) %>% #Choose only the top observation (arbitrary method)\r\n        ungroup() %>%\r\n        arrange(ArtistBio) #Arrange ArtistBio\r\n      \r\n      # For some works, there are several artists from different countries.\r\n      # This code will only list the country of the first artist\r\n      # It will only list countries where there are at least two artists from\r\n      \r\n      \r\n      #Remove everything after commas\r\n      df7$ArtistBio2 <- tolower(df7$ArtistBio)\r\n      df7$ArtistBio2 <- gsub(\",.*\", \"\" , df7$ArtistBio2)\r\n      \r\n      #Remove all parantheses\r\n      df7$ArtistBio2 <- gsub(\"\\\\(\", \"\" , df7$ArtistBio2)\r\n      df7$ArtistBio2 <- gsub(\"\\\\)\", \"\" , df7$ArtistBio2)\r\n      \r\n      #Remove errors in the data with \"various\" for multiple artists\r\n      df7$ArtistBio2 <- gsub(\"various \", \"\" , df7$ArtistBio2)\r\n      df7$ArtistBio2 <- gsub(\"various\", \"\" , df7$ArtistBio2)\r\n    \r\n    #Count artists by country to get simpler dataframe\r\n      df7.1 <- df7 %>% \r\n        group_by(ArtistBio2) %>% \r\n        summarize(count = n()) %>% \r\n        arrange(desc(count))\r\n      df7.1\r\n    \r\n    #Correct simplest mistakes for top countries, and delete remaining\r\n    \r\n      #Rename rows\r\n      df7.1$ArtistBio2[25] <- \"american\"\r\n      df7.1$ArtistBio2[52] <- \"german\"\r\n      df7.1$ArtistBio2[55] <- \"russian\"\r\n      df7.1$ArtistBio2[59] <- \"italian\"\r\n      df7.1$ArtistBio2[63] <- \"british\"\r\n      df7.1$ArtistBio2[70] <- \"british\"\r\n      df7.1$ArtistBio2[74] <- \"french\"\r\n      df7.1$ArtistBio2[79] <- \"cuban\"\r\n      df7.1$ArtistBio2[80] <- \"american\"\r\n      df7.1$ArtistBio2[90] <- \"dutch\"\r\n      df7.1$ArtistBio2[92] <- \"zimbabwean\"\r\n    \r\n      #Rows to be deleted\r\n      delete <- c(7, 71, 77)\r\n      df7.1 <- df7.1[-delete, ]\r\n      \r\n      #Filter away rows that have too few observations (but many errors) and regroup\r\n      df7.1 <-  df7.1 %>% \r\n                filter(count>=3) %>% \r\n                ungroup() %>% \r\n                group_by(ArtistBio2) %>%\r\n                summarize(count = sum(as.numeric(count))) %>% \r\n                arrange(desc(count))\r\n      names(df7.1)[1] <- \"demonymic\"\r\n      \r\n      #Succes rate\r\n      paste(\"Succesfully isolated the artist origins of \", sum(df7.1$count)/nrow(df6)*100,\"% of total artworks\")\r\n      \r\n      #Get relation between country names and what people from there are called\r\n      \r\n        #Download relation table from wikipedia\r\n        url <- \"http://en.wikipedia.org/wiki/List_of_adjectival_and_demonymic_forms_for_countries_and_nations\"\r\n        tabs <- GET(url)\r\n        tabs <- readHTMLTable(rawToChar(tabs$content), stringsAsFactors = F, header = TRUE)\r\n        countries <- as.data.frame(tabs[1], header=TRUE)\r\n        \r\n        #Clean table as data.frame\r\n        #Fix headers\r\n        names(countries) <- c(\"country_name\", \"demonymic\", \"Colloquial\", \"NA\")\r\n        countries <- countries[-1, ]\r\n        #Filter\r\n        countries <- select(countries, country_name, demonymic)\r\n        #Clean\r\n        countries$demonymic <- tolower(countries$demonymic)\r\n        #Remove Wiki-brackets\r\n        countries$demonymic <- gsub(\"\\\\[.*\\\\]\", \"\", x = countries$demonymic) #Remove Wiki-links\r\n        countries$country_name <- gsub(\"\\\\[.*\\\\]\", \"\", x = countries$country_name) #Remove Wiki-links\r\n        countries[ ,3:5] <- str_split_fixed(countries$demonymic, \", \", 3) #Split several options into different rows\r\n\r\n        \r\n      #Join datasets\r\n      df7.2 <- left_join(df7.1, countries, by=c(\"demonymic\"=\"V3\"))\r\n      \r\n      #Fix single countries that are not correctly matched\r\n      df7.2$country_name[1] <- \"USA\"\r\n      df7.2$country_name[27] <- \"USA\"\r\n      df7.2$country_name[36] <- \"South Korea\"\r\n      df7.2$country_name[38] <- \"Yugoslavia\"\r\n      df7.2$country_name[40] <- \"Scotland\"\r\n      df7.2$country_name[53] <- \"New Zealand\"\r\n      df7.2$country_name[61] <- \"Japan\"\r\n      df7.2$country_name[65] <- \"Taiwan\"\r\n      df7.2$country_name[73] <- \"Luxembourg\"\r\n\r\n      \r\n      #Get world map coordinates\r\n      map=map_data(\"world\")\r\n      names(map)=c(\"lon\", \"lat\", \"group\", \"order\", \"Country\", \"subregion\")\r\n      \r\n      #Join datasets\r\n      df7.3 <- left_join(map, df7.2, by=c(\"Country\"=\"country_name\"))\r\n      \r\n      #Plot world map\r\n      p <- ggplot(df7.3, aes(x=lon, y=lat, group=group, fill=df7.3$count))+\r\n        geom_polygon()+\r\n        scale_fill_gradient(\"\", trans=\"log10\")+ \r\n        ggtitle(\"World map: MoMA artworks\")+\r\n        theme(plot.title = element_text(lineheight=.8, face=\"bold\"))\r\n      p\r\n      \r\n## 8: The five largest and five smallest paintings in MoMA's collection\r\n      \r\n      df8 <- df %>%\r\n        select(Dimensions, Title)%>%\r\n        filter(!is.na(Dimensions) & Dimensions!=\"\")%>%\r\n        arrange(Dimensions)\r\n      \r\n      ##Isolates the numbers from within the parenthesis, the dimensions in cms##\r\n      df8$dim <- sub(\".*?\\\\(([\\\\d]+\\\\.?[\\\\d]*)\\\\s[xX\\xD7]\\\\s([\\\\d]+\\\\.?[\\\\d]*).*\", \r\n                     \"\\\\1 \\\\2\", df8$Dimensions, perl = TRUE)\r\n      \r\n      ##Removes irregular observations and splits string in height and width##\r\n      df8 <- filter(df8, grepl(\"[\\\\d]+\\\\.?[\\\\d]*\\\\s[\\\\d]+\\\\.?[\\\\d]*$\", df8$dim, perl = TRUE))\r\n      df8 <- filter(df8, grepl(\"^[\\\\d.]*\\\\s[\\\\d.]+$\", df8$dim, perl = TRUE))\r\n      df8$Height <- as.numeric(str_split_fixed(df8$dim, \" \", 2)[,1])\r\n      df8$Width <- as.numeric(str_split_fixed(df8$dim, \" \", 2)[,2])\r\n      \r\n      ##Creates size-vector##\r\n      df8$size <- df8$Height * df8$Width\r\n      \r\n      ##Prints the five largest and five smallest artworks##\r\n      df8 <- df8 %>%\r\n        group_by(size) %>%\r\n        arrange(-size) %>%\r\n        ungroup %>% \r\n        select(Title, Height, Width, size)\r\n      \r\n      #Five smallest artworks in MoMA\r\n      head(df8, n=5)\r\n      \r\n      #Five largest artworks in MoMA\r\n      df8 <- df8 %>% arrange(desc(size)) \r\n      head(df8, n=5)\r\n      \r\n"}}