Step 1. Create Tokenized Test Document Set - Aster Analytics

CREATE fact TABLE tfidf_token2 DISTRIBUTE BY HASH(docid) AS
  SELECT * FROM nGram (
    ON tfidf_test
    TextColumn ('content')
    Delimiter (' ')
    Grams ('1')
    Overlapping ('false')
    ToLowerCase ('true')
    Punctuation ('\[.,?\!\]')
    Reset ('\[.,?\!\]')
    Total ('false')
    Accumulate ('docid')
  );