ContinuousVariable or a DiscreteVariable.
* This class violates object orientation in that the underlying tetradMatrix matrix is
* retrievable using the getDoubleData() method. This is allowed so that
* external calculations may be performed on large datasets without having to
* allocate extra memory. If this matrix needs to be modified externally, please
* consider making a copy of it first, using the TetradMatrix copy() method.
* n
* The tetradMatrix set may be given a name; this name is not used internally.
* * The tetradMatrix set has a list of variables associated with it, as described above. * This list is coordinated with the stored tetradMatrix, in that tetradMatrix for the i'th * variable will be in the i'th column. *
* A subset of variables in the tetradMatrix set may be designated as selected. This
* selection set is stored with the tetradMatrix set and may be manipulated using the
* select and deselect methods.
// *
// * A multiplicity m_i may be associated with each case c_i in the dataset, which // * is interpreted to mean that that c_i occurs m_i times in the dataset. *
* Knowledge may be associated with the tetradMatrix set, using the
* setKnowledge method. This knowledge is not used internally to
* the tetradMatrix set, but it may be retrieved by algorithm and used.
*
* This tetradMatrix set replaces an earlier Minitab-style DataSet class. The reasons * for replacement are as follows. *
*
tetradMatrix.
*
* @serial
*/
private Listindex to column i + 1,
* and inserting a column of missing values at column i.
*
* @throws IllegalArgumentException if the variable already exists in the
* dataset.
*/
public final void addVariable(Node variable) {
if (variables.contains(variable)) {
throw new IllegalArgumentException("Expecting a new variable: " + variable);
}
variables.add(variable);
resize(tetradMatrix.rows(), variables.size());
int col = tetradMatrix.columns() - 1;
for (int i = 0; i < tetradMatrix.rows(); i++) {
tetradMatrix.set(i, col, Double.NaN);
}
}
/**
* Adds the given variable to the dataset, increasing the number of
* columns by one, moving columns i >= index to column i + 1,
* and inserting a column of missing values at column i.
*/
public final void addVariable(int index, Node variable) {
if (variables.contains(variable)) {
throw new IllegalArgumentException("Expecting a new variable.");
}
if (index < 0 || index > variables.size()) {
throw new IndexOutOfBoundsException("Index must in (0, #vars).");
}
variables.add(index, variable);
resize(tetradMatrix.rows(), variables.size());
TetradMatrix _data =
new TetradMatrix(tetradMatrix.rows(), tetradMatrix.columns() + 1);
for (int j = 0; j < tetradMatrix.columns() + 1; j++) {
if (j < index) {
for (int i = 0; i < tetradMatrix.rows(); i++) {
_data.set(i, j, tetradMatrix.get(i, j));
}
} else if (j == index) {
for (int i = 0; i < tetradMatrix.rows(); i++) {
_data.set(i, j, Double.NaN);
}
} else {
for (int i = 0; i < tetradMatrix.rows(); i++) {
_data.set(i, j, tetradMatrix.get(i, j - 1));
}
}
}
}
/**
* @return the variable at the given column.
*/
public final Node getVariable(int col) {
return variables.get(col);
}
/**
* @return the index of the column of the given variable. You can also get
* this by calling getVariables().indexOf(variable).
*/
public final int getColumn(Node variable) {
return variables.indexOf(variable);
}
/**
* Changes the variable for the given column from from to
* to. Supported currently only for discrete variables.
*
* @throws IllegalArgumentException if the given change is not supported.
*/
@SuppressWarnings({"ConstantConditions"})
public final void changeVariable(Node from, Node to) {
if (!(from instanceof DiscreteVariable &&
to instanceof DiscreteVariable)) {
throw new IllegalArgumentException(
"Only discrete variables supported.");
}
DiscreteVariable _from = (DiscreteVariable) from;
DiscreteVariable _to = (DiscreteVariable) to;
int col = variables.indexOf(_from);
ListStatistic.covariance() in the COLT matrix library, so it
* inherits the handling of missing values from that library--that is, any
* off-diagonal correlation involving a column with a missing value is
* Double.NaN, although all of the on-diagonal elements are 1.0. If that's
* not the desired behavior, missing values can be removed or imputed
* first.
*/
public final TetradMatrix getCorrelationMatrix() {
if (!isContinuous()) {
throw new IllegalStateException("Not a continuous tetradMatrix set.");
}
TetradMatrix cov = new TetradMatrix(tetradMatrix.columns(), tetradMatrix.columns());
for (int i = 0; i < tetradMatrix.columns(); i++) {
for (int j = i; j < tetradMatrix.columns(); j++) {
double correlation = StatUtils.correlation(tetradMatrix.getColumn(i).toArray(), tetradMatrix.getColumn(j).toArray());
cov.set(i, j, correlation);
cov.set(j, i, correlation);
}
}
return cov;
// return Statistic.correlation(Statistic.covariance(tetradMatrix));
}
/**
* @return the covariance matrix for this dataset. Defers to
* Statistic.covariance() in the COLT matrix library, so it
* inherits the handling of missing values from that library--that is, any
* covariance involving a column with a missing value is Double.NaN. If
* that's not the desired behavior, missing values can be removed or imputed
* first.
*/
public final TetradMatrix getCovarianceMatrix() {
if (!isContinuous()) {
throw new IllegalStateException("Not a continuous data set.");
}
return DataUtils.cov(tetradMatrix);
// return DataUtils.covErich(tetradMatrix);
}
/**
* @return the value at the given row and column, rounded to the nearest
* integer, or DiscreteVariable.MISSING_VALUE if the value is missing.
*/
public final int getInt(int row, int column) {
double value = tetradMatrix.get(row, column);
if (Double.isNaN(value)) {
return DiscreteVariable.MISSING_VALUE;
} else {
return (int) Math.round(value);
}
}
/**
* @return the double value at the given row and column. For discrete
* variables, this returns an int cast to a double. The double value at the
* given row and column may be missing, in which case Double.NaN is
* returned.
*/
public final double getDouble(int row, int column) {
return tetradMatrix.get(row, column);
}
// /**
// * Sets the case multiplier for the given case to the given number (must be
// * >= 1).
// */
// public final void setMultiplier(int caseNumber, int multiplier) {
// if (caseNumber < 0) {
// throw new IllegalArgumentException(
// "Case numbers must be >= 0: " + caseNumber);
// }
//
// if (multiplier < 0) {
// throw new IllegalArgumentException(
// "Multipliers must be >= 0: " + multiplier);
// }
//
// if (multiplier == 1) {
// getMultipliers().remove(caseNumber);
// } else {
// getMultipliers().put(caseNumber, multiplier);
// }
// }
/**
* @return a string, suitable for printing, of the dataset. Lines are
* separated by '\n', tokens in the line by whatever character is set in the
* setOutputDelimiter() method. The list of variables is printed
* first, followed by one line for each case.
*
* This method should probably not be used for saving to files. If that's
* your goal, use the DataSavers class instead.
* @see #setOutputDelimiter(Character)
* @see DataWriter
*/
public final String toString() {
StringBuilder buf = new StringBuilder();
List variables = getVariables();
buf.append("\n");
if (isLineNumbersWritten()) {
buf.append("\t");
}
for (int i = 0; i < getNumColumns(); i++) {
buf.append(variables.get(i));
if (i < getNumColumns() - 1) {
buf.append(outputDelimiter);
}
}
buf.append("\n");
for (int i = 0; i < getNumRows(); i++) {
if (isLineNumbersWritten()) {
buf.append(i + 1).append(".\t");
}
for (int j = 0; j < getNumColumns(); j++) {
Node variable = getVariable(j);
if (variable instanceof ContinuousVariable) {
if (Double.isNaN(getDouble(i, j))) {
buf.append("*");
} else {
buf.append(getNumberFormat().format(getDouble(i, j)));
}
if (j < getNumColumns() - 1) {
buf.append(outputDelimiter);
}
} else if (variable instanceof DiscreteVariable) {
DiscreteVariable _variable = (DiscreteVariable) variable;
int value = getInt(i, j);
if (value == -99) {
buf.append("*");
} else {
String category = _variable.getCategory(value);
if (category.indexOf((int) outputDelimiter) == -1) {
buf.append(category);
} else {
buf.append("\"").append(category).append("\"");
}
}
if (j < getNumColumns() - 1) {
buf.append(outputDelimiter);
}
} else {
throw new IllegalStateException(
"Expecting either a continuous " +
"variable or a discrete variable: " + variable);
}
}
buf.append("\n");
}
buf.append("\n");
if (knowledge != null && !knowledge.isEmpty()) {
buf.append(knowledge);
}
return buf.toString();
}
/**
* @return a copy of the underlying COLT TetradMatrix matrix, containing
* all of the tetradMatrix in this dataset, discrete tetradMatrix included. Discrete tetradMatrix
* will be represented by ints cast to doubles. Rows in this matrix are
* cases, and columns are variables. The list of variable, in the order in
* which they occur in the matrix, is given by getVariables().
*
* If isMultipliersCollapsed() returns false, multipliers in the dataset are
* first expanded before returning the matrix, so the number of rows in the
* returned matrix may not be the same as the number of rows in this
* dataset.
* @throws IllegalStateException if this is not a continuous tetradMatrix set.
* @see #getVariables
// * @see #isMulipliersCollapsed()
*/
public final TetradMatrix getDoubleData() {
// return tetradMatrix.copy();
// if (!isMulipliersCollapsed()) {
// CaseExpander expander = new CaseExpander();
// return ((ColtDataSet) expander.filter(this)).tetradMatrix;
// } else {
return tetradMatrix;
// }
}
// public final TetradMatrix getDoubleDataNoCopy() {
// return tetradMatrix;
// }
/**
* @return a new tetradMatrix set in which the the column at indices[i] is placed at
* index i, for i = 0 to indices.length - 1. (Moved over from Purify.)
*/
public final DataSet subsetColumns(int indices[]) {
List variables = getVariables();
List _variables = new LinkedList<>();
for (int index : indices) {
if (index != -1) {
_variables.add(variables.get(index));
}
}
int[] rows = new int[tetradMatrix.rows()];
for (int i = 0; i < rows.length; i++) {
rows[i] = i;
}
TetradMatrix _data = tetradMatrix.getSelection(rows, indices).copy();
ColtDataSet _dataSet = new ColtDataSet(0, new LinkedList());
_dataSet.tetradMatrix = _data;
// _dataSet.name = name + "_copy";
_dataSet.name = name;
_dataSet.variables = _variables;
_dataSet.selection = new HashSet<>();
// _dataSet.multipliers = new HashMap<>(multipliers);
// Might have to delete some knowledge.
_dataSet.knowledge = knowledge.copy();
return _dataSet;
}
public final DataSet subsetRows(int rows[]) {
int cols[] = new int[this.tetradMatrix.columns()];
for (int i = 0; i < cols.length; i++) {
cols[i] = i;
}
ColtDataSet _data = new ColtDataSet(this);
_data.tetradMatrix = this.tetradMatrix.getSelection(rows, cols);
return _data;
}
/**
* Shifts the given column
*/
public final void shiftColumnDown(int row, int col, int numRowsShifted) {
// Find last row that does not consist entirely of missing values.
if (row >= getNumRows() || col >= getNumColumns()) {
throw new IllegalArgumentException("Out of range: row = " + row + " col = " + col);
}
int lastRow = -1;
for (int i = getNumRows() - 1; i >= row; i--) {
if (!Double.isNaN(tetradMatrix.get(i, col))) {
lastRow = i;
break;
}
}
if (lastRow == -1) {
return;
}
resize(getNumRows() + numRowsShifted, getNumColumns());
for (int i = getNumRows() - 1; i >= row + numRowsShifted; i--) {
tetradMatrix.set(i, col, tetradMatrix.get(i - numRowsShifted, col));
tetradMatrix.set(i - numRowsShifted, col, Double.NaN);
}
}
/**
* Removes the given columns from the tetradMatrix set.
*/
public final void removeCols(int[] cols) {
int[] rows = new int[tetradMatrix.rows()];
for (int i = 0; i < tetradMatrix.rows(); i++) {
rows[i] = i;
}
int[] retainedCols = new int[variables.size() - cols.length];
int i = -1;
for (int j = 0; j < variables.size(); j++) {
if (Arrays.binarySearch(cols, j) < 0) {
retainedCols[++i] = j;
}
}
List retainedVars = new LinkedList<>();
for (int retainedCol : retainedCols) {
retainedVars.add(variables.get(retainedCol));
}
tetradMatrix = tetradMatrix.getSelection(rows, retainedCols).copy();
variables = retainedVars;
selection = new HashSet<>();
// multipliers = new HashMap<>(multipliers);
knowledge = knowledge.copy(); // Might have to delete some knowledge.
}
/**
* Removes the given rows from the tetradMatrix set.
*/
public final void removeRows(int[] selectedRows) {
int[] cols = new int[tetradMatrix.columns()];
for (int i = 0; i < tetradMatrix.columns(); i++) {
cols[i] = i;
}
int[] retainedRows = new int[tetradMatrix.rows() - selectedRows.length];
int i = -1;
for (int j = 0; j < tetradMatrix.rows(); j++) {
if (Arrays.binarySearch(selectedRows, j) < 0) {
retainedRows[++i] = j;
}
}
tetradMatrix = tetradMatrix.getSelection(retainedRows, cols).copy();
selection = new HashSet<>();
// multipliers = new HashMap<>(multipliers);
knowledge = knowledge.copy(); // Might have to delete some knowledge.
}
/**
* @return true iff obj is a continuous RectangularDataSet with
* corresponding variables of the same name and corresponding tetradMatrix values
* equal, when rendered using the number format at NumberFormatUtil.getInstance().getNumberFormat().
*/
public final boolean equals(Object obj) {
NumberFormat nf = NumberFormatUtil.getInstance().getNumberFormat();
if (obj == this) {
return true;
}
if (!(obj instanceof DataSet)) {
return false;
}
DataSet _dataSet = (DataSet) obj;
List dataVars = _dataSet.getVariables();
if (getVariables().size() != dataVars.size()) {
return false;
}
dataVars = GraphUtils.replaceNodes(dataVars, getVariables());
for (int i = 0; i < getVariables().size(); i++) {
Node node = getVariables().get(i);
Node _node = dataVars.get(i);
if (!node.equals(_node)) {
return false;
}
}
if (!(_dataSet.getNumRows() == getNumRows())) {
return false;
}
for (int i = 0; i < getNumRows(); i++) {
for (int j = 0; j < getNumColumns(); j++) {
Node variable = getVariable(j);
if (variable instanceof ContinuousVariable) {
final double d1 = getDouble(i, j);
final double d2 = _dataSet.getDouble(i, j);
if (Double.isNaN(d1) && !Double.isNaN(d2)) {
return false;
}
if (!Double.isNaN(d1) && Double.isNaN(d2)) {
return false;
}
if (Double.isNaN(d1) && Double.isNaN(d2)) {
return true;
}
if (Math.abs(d1 - d2) > Math.pow(10, -(nf.getMaximumFractionDigits()))) {
return false;
}
// Parsing and rendering is expensive.
// double value = Double.parseDouble(nf.format(getDouble(i, j)));
// double _value = Double.parseDouble(nf.format(_dataSet.getDouble(i, j)));
//
// if (Math.abs(value - _value) > 0.0) {
// return false;
// }
} else {
double value = getInt(i, j);
double _value = _dataSet.getInt(i, j);
if (!(value == _value)) {
return false;
}
}
}
}
return true;
}
@Override
public DataSet copy() {
return new ColtDataSet(this);
}
@Override
public DataSet like() {
return new ColtDataSet(getNumRows(), variables);
}
public void setNumberFormat(NumberFormat nf) {
if (nf == null) {
throw new NullPointerException();
}
this.nf = nf;
}
/**
* Sets the character ('\t', ' ', ',', for instance) that is used to delimit
* tokens when the tetradMatrix set is printed out using the toString() method.
*
* @see #toString
*/
public void setOutputDelimiter(Character character) {
this.outputDelimiter = character;
}
/**
* Randomly permutes the rows of the dataset.
*/
public void permuteRows() {
List permutation = new ArrayList<>();
for (int i = 0; i < getNumRows(); i++) {
permutation.add(i);
}
Collections.shuffle(permutation);
TetradMatrix data2 = tetradMatrix.like();
for (int i = 0; i < getNumRows(); i++) {
for (int j = 0; j < getNumColumns(); j++) {
data2.set(i, j, tetradMatrix.get(permutation.get(i), j));
}
}
this.tetradMatrix = data2;
}
//===============================PRIVATE METHODS=====================//
private void setIntPrivate(int row, int col, int value) {
if (value == -99) {
tetradMatrix.set(row, col, Double.NaN);
} else {
tetradMatrix.set(row, col, value);
}
}
/**
* Resizes the tetradMatrix to the given dimensions. Data that does not fall within
* the new dimensions is lost, and positions in the redimensioned tetradMatrix that
* have no correlates in the old tetradMatrix are set to missing (that is,
* Double.NaN).
*
* @param rows The number of rows in the redimensioned tetradMatrix.
* @param cols The number of columns in the redimensioned tetradMatrix.
*/
private void resize(int rows, int cols) {
TetradMatrix _data = new TetradMatrix(rows, cols);
for (int i = 0; i < _data.rows(); i++) {
for (int j = 0; j < _data.columns(); j++) {
if (i < tetradMatrix.rows() && j < tetradMatrix.columns()) {
_data.set(i, j, tetradMatrix.get(i, j));
} else {
_data.set(i, j, Double.NaN);
}
}
}
tetradMatrix = _data;
}
// /**
// * @return the set of case multipliers..
// */
// private Map getMultipliers() {
// return multipliers;
// }
/**
* Adds semantic checks to the default deserialization method. This method
* must have the standard signature for a readObject method, and the body of
* the method must begin with "s.defaultReadObject();". Other than that, any
* semantic checks can be specified and do not need to stay the same from
* version to version. A readObject method of this form may be added to any
* class, even if Tetrad sessions were previously saved out using a version
* of the class that didn't include it. (That's what the
* "s.defaultReadObject();" is for. See J. Bloch, Effective Java, for help.
*
* @throws java.io.IOException
* @throws ClassNotFoundException
*/
private void readObject(ObjectInputStream s)
throws IOException, ClassNotFoundException {
s.defaultReadObject();
}
/**
* @return the set of selected nodes, creating a new set if necessary.
*/
private Set getSelection() {
if (selection == null) {
selection = new HashSet<>();
}
return selection;
}
/**
* Attempts to translate element into a double value, returning
* it if successful, otherwise throwing an exception. To be successful, the
* object must be either a Number or a String.
*
* @throws IllegalArgumentException if the translation cannot be made. The
* reason is in the message.
*/
private static double getValueFromObjectContinuous(Object element) {
if ("*".equals(element) || "".equals(element)) {
return ContinuousVariable.getDoubleMissingValue();
} else if (element instanceof Number) {
return ((Number) element).doubleValue();
} else if (element instanceof String) {
try {
return Double.parseDouble((String) element);
} catch (NumberFormatException e) {
return ContinuousVariable.getDoubleMissingValue();
}
} else {
throw new IllegalArgumentException(
"The argument 'element' must be " +
"either a Number or a String.");
}
}
/**
* Attempts to translate element into an int value, returning
* it if successful, otherwise throwing an exception. To be successful, the
* object must be either a Number or a String.
*
* @throws IllegalArgumentException if the translation cannot be made. The
* reason is in the message.
*/
private int getValueFromObjectDiscrete(Object element,
DiscreteVariable variable) {
if ("*".equals(element) || "".equals(element)) {
return DiscreteVariable.MISSING_VALUE;
}
if (variable.isAccommodateNewCategories()) {
if (element instanceof Number) {
int index = ((Number) element).intValue();
if (!variable.checkValue(index)) {
if (index >= variable.getNumCategories()) {
accomodateIndex(variable, index);
} else {
throw new IllegalArgumentException("Variable " + variable +
" is not accepting " +
"new categories. Problem category is " + ".");
}
}
return index;
} else if (element instanceof String) {
String label = (String) element;
if ("".equals(label)) {
throw new IllegalArgumentException(
"Blank category names not permitted.");
}
variable = accomodateCategory(variable, label);
int index = variable.getIndex(label);
if (index == -1) {
throw new IllegalArgumentException(
"Not a category for this variable: " + index);
}
return index;
} else {
throw new IllegalArgumentException(
"The argument 'element' must be " +
"either a Number or a String.");
}
} else {
if (element instanceof Number) {
int index = ((Number) element).intValue();
if (!variable.checkValue(index)) {
return DiscreteVariable.MISSING_VALUE;
}
return index;
} else if (element instanceof String) {
String label = (String) element;
int index = variable.getIndex(label);
if (index == -1) {
return DiscreteVariable.MISSING_VALUE;
}
return index;
} else {
throw new IllegalArgumentException(
"The argument 'element' must be " +
"either a Number or a String.");
}
}
}
/**
* If the given category is not already a category for a cagetory, augments
* the range of category by one and sets the category of the new value to
* the given category.
*/
private DiscreteVariable accomodateCategory(DiscreteVariable variable,
String category) {
if (category == null) {
throw new NullPointerException();
}
List categories = variable.getCategories();
if (!categories.contains(category)) {
List newCategories = new LinkedList<>(categories);
newCategories.add(category);
DiscreteVariable newVariable =
new DiscreteVariable(variable.getName(), newCategories);
changeVariable(variable, newVariable);
return newVariable;
}
return variable;
}
/**
* Increases the number of categories if necessary to make sure that this
* variable has the given index.
*/
private void accomodateIndex(DiscreteVariable variable, int index) {
if (!variable.isAccommodateNewCategories()) {
throw new IllegalArgumentException("This variable is not set " +
"to accomodate new categories.");
}
if (index >= variable.getNumCategories()) {
adjustCategories(variable, index + 1);
}
}
/**
* Adjusts the size of the categories list to match the getModel number of
* categories. If the list is too short, it is padded with default
* categories. If it is too long, the extra categories are removed.
*/
private void adjustCategories(DiscreteVariable variable,
int numCategories) {
List categories =
new LinkedList<>(variable.getCategories());
List newCategories = new LinkedList<>(categories);
if (categories.size() > numCategories) {
for (int i = variable.getCategories().size() - 1;
i >= numCategories; i++) {
newCategories.remove(i);
}
} else if (categories.size() < numCategories) {
for (int i = categories.size(); i < numCategories; i++) {
String category = DataUtils.defaultCategory(i);
if (categories.contains(category)) {
continue;
}
newCategories.add(category);
}
}
DiscreteVariable to =
new DiscreteVariable(variable.getName(), newCategories);
changeVariable(variable, to);
}
/**
* @return the number format, which by default is the one at
* NumberFormatUtil.getInstance().getNumberFormat(), but can be
* set by the user if desired.
* @see #setNumberFormat(java.text.NumberFormat)
*/
public NumberFormat getNumberFormat() {
if (nf == null) {
nf = NumberFormatUtil.getInstance().getNumberFormat();
}
return nf;
// return new DecimalFormat("0");
}
private boolean isLineNumbersWritten() {
return lineNumbersWritten;
}
public void setLineNumbersWritten(boolean lineNumbersWritten) {
this.lineNumbersWritten = lineNumbersWritten;
}
}