Concatenate multiple files with same headers (and only keep one header line in the output file)

awk '
    FNR==1 && NR!=1 { while (/^<header>/) getline; }
    1 {print}
' file*.txt >all.txt

Note: the /^<header>/ part need to be changed to adapt to whatever the actual header is.

R - conversion

Don't directly convert factor to numeric! Change to characters first!

df$a = as.numeric(df$a) ---> NOT GOOD!

df$a = as.numeric(as.character(df$a)) ---> GOOD!

convert the first tab into pip

cat SOC.csv | sed -e "s/\t/$(printf '|')/"  > SOC.pip

Sanitize U.S. States Names

# load data

StateData = read.csv('65States.pip', sep="|", col.names = c("FullName", "Abbr"))

StateFullName = toupper(StateData$FullName)

StateAbbr = as.vector(StateData$Abbr)

# define a function

sanitizeState = function(inputcol, StateFullName, StateAbbr){

  match = amatch(inputcol, StateFullName, maxDist=1)

  inputcol[!is.na(match)] = StateAbbr[na.omit(match)]

  return (inputcol)

}

# use the function

df$State = sanitizeState(gls$State,StateFullName, StateAbbr )

python - read large csv into pandas by chunks

chunks=pd.read_table('filename',  chunksize=500000)
df=pd.DataFrame()
df=pd.concat((chunk==1) for chunk in chunks)

remove deplicates

To remove duplicated rows:


awk '!seen[$0]++' <filename>

To remove rows with duplicated field (say $1 is ID and need to remove the entire row if ID is duplicated):

awk '!seen[$1]++' <filename>

filter a file based on tokens in another file

BEGIN{

  FS="|"

  OFS="|"

  while ((getline < (“Token_list_file.csv")) > 0) {

  id[$1]=$1;

  }

}

{

  appid = $1;

  if(appid in id) {print $0;}

}