<SYSTEM_TASK:>
Given the following text description, write Python code to implement the functionality described below step by step
<END_TASK>
<USER_TASK:>
Description:
 target data for feature selection
average all data for each compound
Step1:  target data for training
filter out the relevant data for each compound
<ASSISTANT_TASK:>
Python Code:
# load the training data 
data = pd.read_csv(os.path.abspath('__file__' + "/../../../data/TrainSet.txt"),sep='\t')
data.drop(['Intensity','Odor','Replicate','Dilution'],axis=1, inplace=1)
data.columns = ['#oID', 'individual'] + list(data.columns)[2:]
data.head()
# load leaderboard data and reshape them to match the training data
LB_data_high = pd.read_csv(os.path.abspath('__file__' + "/../../../data/LBs1.txt"),sep='\t')
LB_data_high = LB_data_high.pivot_table(index=['#oID','individual'],columns='descriptor',values='value')
LB_data_high.reset_index(level=[0,1],inplace=1)
LB_data_high.rename(columns={' CHEMICAL':'CHEMICAL'}, inplace=True)
LB_data_high = LB_data_high[data.columns]
LB_data_high.head()
# load leaderboard low intensity data and reshape them to match the training data
LB_data_low = pd.read_csv(os.path.abspath('__file__' + "/../../../data/leaderboard_set_Low_Intensity.txt"),sep='\t')
LB_data_low = LB_data_low.pivot_table(index=['#oID','individual'],columns='descriptor',values='value')
LB_data_low.reset_index(level=[0,1],inplace=1)
LB_data_low.rename(columns={' CHEMICAL':'CHEMICAL'}, inplace=True)
LB_data_low = LB_data_low[data.columns]
LB_data_low.head()
# put them all together
selection_data = pd.concat((data,LB_data_high,LB_data_low),ignore_index=True)
# replace descriptor data with np.nan if intensity is zero
for descriptor in [u'VALENCE/PLEASANTNESS', u'BAKERY', u'SWEET', u'FRUIT', u'FISH',
      u'GARLIC', u'SPICES', u'COLD', u'SOUR', u'BURNT', u'ACID', u'WARM',
      u'MUSKY', u'SWEATY', u'AMMONIA/URINOUS', u'DECAYED', u'WOOD',
      u'GRASS', u'FLOWER', u'CHEMICAL']:
    selection_data[descriptor][(selection_data['INTENSITY/STRENGTH'] == 0)] = np.nan
selection_data = selection_data.groupby('#oID').mean()
selection_data.reset_index(inplace=1)
selection_data.drop('individual',1,inplace=1)
selection_data.head()
selection_data = pd.concat((selection_data,selection_data)).sort('#oID')
selection_data.index=range(len(selection_data))
# load test CIDs to remove them from russdata
with open(os.path.abspath('__file__' + "/../../../data/CID_testset.txt")) as f: 
    content = f.readlines()
test_CIDs = list(content)  
test_CIDs = [int(x) for x in test_CIDs]
intensity = russdata[~russdata.CID.isin(test_CIDs)]
intensity.index = range(len(intensity))
print((intensity.CID != selection_data['#oID']).sum())
intensity = intensity['INTENSITY/STRENGTH']
selection_data['INTENSITY/STRENGTH'] = intensity
selection_data.head()
selection_data.shape 
selection_data.to_csv('targets_for_feature_selection.csv')
selection_data.head()
Explanation: target data for feature selection
average all data for each compound
End of explanation
# load the train data 
data = pd.read_csv(os.path.abspath('__file__' + "/../../../data/TrainSet.txt"),sep='\t')
data.drop(['Odor','Replicate'],axis=1, inplace=1)
data.columns = [u'#oID','Intensity','Dilution', u'individual', u'INTENSITY/STRENGTH', u'VALENCE/PLEASANTNESS', u'BAKERY', u'SWEET', u'FRUIT', u'FISH', u'GARLIC', u'SPICES', u'COLD', u'SOUR', u'BURNT', u'ACID', u'WARM', u'MUSKY', u'SWEATY', u'AMMONIA/URINOUS', u'DECAYED', u'WOOD', u'GRASS', u'FLOWER', u'CHEMICAL']
data.head()
#load LB data
LB_data_high = pd.read_csv(os.path.abspath('__file__' + "/../../../data/LBs1.txt"),sep='\t')
LB_data_high = LB_data_high.pivot_table(index=['#oID','individual'],columns='descriptor',values='value')
LB_data_high.reset_index(level=[0,1],inplace=1)
LB_data_high.rename(columns={' CHEMICAL':'CHEMICAL'}, inplace=True)
LB_data_high['Dilution'] = '1/1,000 '
LB_data_high['Intensity'] = 'high '
LB_data_high = LB_data_high[data.columns]
LB_data_high.head()
# put them together
data = pd.concat((data,LB_data_high),ignore_index=True)
# replace descriptor data with np.nan if intensity is zero
for descriptor in [u'VALENCE/PLEASANTNESS', u'BAKERY', u'SWEET', u'FRUIT', u'FISH',
      u'GARLIC', u'SPICES', u'COLD', u'SOUR', u'BURNT', u'ACID', u'WARM',
      u'MUSKY', u'SWEATY', u'AMMONIA/URINOUS', u'DECAYED', u'WOOD',
      u'GRASS', u'FLOWER', u'CHEMICAL']:
    data[descriptor][(data['INTENSITY/STRENGTH'] == 0)] = np.nan
# average the duplicates 
data = data.groupby(['individual','#oID','Dilution','Intensity']).mean() 
data.reset_index(level=[2,3], inplace=True) 
# filter out data with high intensity
data = data[data.Intensity == 'high ']
# drop Dilution and Intensity columns
data.drop(['Dilution','Intensity'],inplace=1,axis=1)
data.reset_index(level=[0,1], inplace=True)
data = data.groupby('#oID').mean()
data.shape
data.head()
#duplacate the matrix, add neglog10 (Dilution) and Intensity data from russdata
data.reset_index(inplace=1)
data = pd.concat((data,data)).sort('#oID')
data.index=range(len(data))
# remove test targets from russdata intensity (Nans)
intensity = russdata[~russdata.CID.isin(test_CIDs)]
intensity.index = range(len(intensity))
print((intensity.CID != data['#oID']).sum()) # check if data orders are the same - should get 0 if they are the same
intensity = intensity['INTENSITY/STRENGTH']
data['INTENSITY/STRENGTH'] = intensity
data.head()
data.shape
#save it
data.to_csv('target.csv')
Explanation: target data for training
filter out the relevant data for each compound
End of explanation
<END_TASK>