Example 2: Using TD_TextParser to Get Token Frequency and List of Positions - Analytics Database

InputTable: test_table data

CREATE TABLE test_table (
    id INTEGER, paragraph VARCHAR(100)
);
INSERT INTO test_table (id, paragraph) VALUES(1, 'Programmers program with program, as.as   programming languages a program');
INSERT INTO test_table (id, paragraph) VALUES(2, 'The quick brown fox jumps over the lazy dog');

SELECT Statement

SELECT * from test_table;

Result:

paragraph
-----------------------------------------------------------------------
Programmers program with program, as.as programming languages a program
The quick brown fox jumps over the lazy dog

StopWords table: Custom set of words to be removed when parsing

CREATE TABLE stopwords (word varchar(10));
INSERT INTO stopwords('a');
INSERT INTO stopwords('an');
INSERT INTO stopwords('and');
INSERT INTO stopwords('the');

SELECT Statement

SELECT * from stopwords

Result:

word
-----
the
and
an
a

Query 1 (Tokenizing with default delimiter)

SELECT * FROM TD_TextParser (
ON test_table AS InputTable
USING
TextColumn ('paragraph')
RemoveStopWords ('true')
) as dt ORDER BY 1,4

Result:

Query 2 (Using StopWordsTable, ListPositions, and TokenFrequency)

SELECT * FROM TD_TextParser (
ON test_table AS InputTable
ON stopwords as StopWordsTable DIMENSION
USING
TextColumn ('paragraph')
RemoveStopWords ('true')
DocIDColumn('id')
ListPositions('t')
TokenFrequency('t')
) as dt ORDER BY 1,2

Result:

Query 3 (Using OutputByWords set to false and Delimiter is a blank space)

SELECT * FROM TD_TextParser (
ON test_table AS InputTable
ON stopwords as StopWordsTable DIMENSION
USING
TextColumn ('paragraph')
RemoveStopWords ('true')
Delimiter(' ')
OutputByWord('false')
) as dt ORDER BY 1,2

Result:

Query 4 (Using DelimiterRegex)

SELECT * FROM TD_TextParser (
ON test_table AS InputTable
USING
TextColumn ('paragraph')
RemoveStopWords ('true')
DocIDColumn('id')
DelimiterRegex('[ \t\f\r\n]+')
ListPositions('true')
) as dt ORDER BY 1,4

Result: