Weka - Naive Bayes always gives borderline results - nlp

Question

I am trying to write a text classifier in Weka with Naive Bayes. I have a collection of Foursquare tips as training data with close to 500 of them marked as positive and approximately same marked as negative in an excel file. The input file has two columns with first one being the tip text and second one the marked polarity. I am using AFINN-111.txt to add an attribute to enhance the output. It calculates all the polar words in that tip and gives a final score of all the words. Here is my entire code:
public class DataReader {
static Map<String, Integer> affinMap = new HashMap<String, Integer>();
public List<List<Object>> createAttributeList() {
ClassLoader classLoader = getClass().getClassLoader();
initializeAFFINMap(classLoader);
File inputWorkbook = new File(classLoader
.getResource("Tip_dataset2.xls").getFile());
Workbook w;
Sheet sheet = null;
try {
w = Workbook.getWorkbook(inputWorkbook);
// Get the first sheet
sheet = w.getSheet(0);
} catch (Exception e) {
e.printStackTrace();
}
List<List<Object>> attributeList = new ArrayList<List<Object>>();
for (int i = 1; i < sheet.getRows(); i++) {
String tip = sheet.getCell(0, i).getContents();
tip = tip.replaceAll("'", "");
tip = tip.replaceAll("\"", "");
tip = tip.replaceAll("%", " percent");
tip = tip.replaceAll("#", " ATAUTHOR");
String polarity = getPolarity(sheet.getCell(1, i).getContents());
int affinScore = 0;
String[] arr = tip.split(" ");
for (int j = 0; j < arr.length; j++) {
if (affinMap.containsKey(arr[j].toLowerCase())) {
affinScore = affinScore
+ affinMap.get(arr[j].toLowerCase());
}
}
List<Object> attrs = new ArrayList<Object>();
attrs.add(tip);
attrs.add(affinScore);
attrs.add(polarity);
attributeList.add(attrs);
}
return attributeList;
}
private String getPolarity(String cell) {
if (cell.equalsIgnoreCase("positive")) {
return "positive";
} else {
return "negative";
}
}
private void initializeAFFINMap(ClassLoader classLoader) {
try {
InputStream stream = classLoader
.getResourceAsStream("AFINN-111.txt");
DataInputStream in = new DataInputStream(stream);
BufferedReader br = new BufferedReader(new InputStreamReader(in));
String str;
while ((str = br.readLine()) != null) {
String[] array = str.split("\t");
affinMap.put(array[0], Integer.parseInt(array[1]));
}
in.close();
} catch (Exception e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws Exception {
List<List<Object>> attrList=new DataReader().createAttributeList();
new CreateTrainedModel().createTrainingData(attrList);
}
}
Here is the actual classifier class:
public class CreateTrainedModel {
public void createTrainingData(List<List<Object>> attrList)
throws Exception {
Attribute tip = new Attribute("tip", (FastVector) null);
Attribute affin = new Attribute("affinScore");
FastVector pol = new FastVector(2);
pol.addElement("positive");
pol.addElement("negative");
Attribute polaritycl = new Attribute("polarity", pol);
FastVector inputDataDesc = new FastVector(3);
inputDataDesc.addElement(tip);
inputDataDesc.addElement(affin);
inputDataDesc.addElement(polaritycl);
Instances dataSet = new Instances("dataset", inputDataDesc,
attrList.size());
// Set class index
dataSet.setClassIndex(2);
for (List<Object> onList : attrList) {
Instance in = new Instance(3);
in.setValue((Attribute) inputDataDesc.elementAt(0), onList.get(0)
.toString());
in.setValue((Attribute) inputDataDesc.elementAt(1),
Integer.parseInt(onList.get(1).toString()));
in.setValue((Attribute) inputDataDesc.elementAt(2), onList.get(2)
.toString());
dataSet.add(in);
}
Filter f = new StringToWordVector();
f.setInputFormat(dataSet);
dataSet = Filter.useFilter(dataSet, f);
Classifier model = (Classifier) new NaiveBayes();
try {
model.buildClassifier(dataSet);
} catch (Exception e1) { // TODO Auto-generated catch block
e1.printStackTrace();
}
ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(
"FS-TipsNaiveBayes.model"));
oos.writeObject(model);
oos.flush();
oos.close();
FastVector fvWekaAttributes1 = new FastVector(3);
fvWekaAttributes1.addElement(tip);
fvWekaAttributes1.addElement(affin);
Instance in = new Instance(3);
in.setValue((Attribute) fvWekaAttributes1.elementAt(0),
"burger here is good");
in.setValue((Attribute) fvWekaAttributes1.elementAt(1), 0);
Instances testSet = new Instances("dataset", fvWekaAttributes1, 1);
in.setDataset(testSet);
double[] fDistribution = model.distributionForInstance(in);
System.out.println(fDistribution);
}
}
The problem I am facing is with any input the output distribution is always in the range of [0.52314376998377, 0.47685623001622995]. And it is always more towards the positive than the negative. These figures do not change drastically. Any idea what wrong am I doing?

Answer 1

I didn't read your code, but one thing I can say is that the AFFIN score is normalized between a certain range. If your output is more towards a positive range then you need to change your classification cost function, because it is overfitting your data.