001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.Serializable; 016import java.lang.reflect.Array; 017import java.text.Format; 018import java.util.ArrayList; 019import java.util.Arrays; 020import java.util.List; 021import java.util.concurrent.ConcurrentHashMap; 022import java.util.concurrent.ConcurrentMap; 023 024import org.eclipse.january.DatasetException; 025import org.eclipse.january.IMonitor; 026import org.eclipse.january.MetadataException; 027import org.eclipse.january.metadata.ErrorMetadata; 028import org.eclipse.january.metadata.MetadataFactory; 029import org.eclipse.january.metadata.MetadataType; 030import org.eclipse.january.metadata.StatisticsMetadata; 031import org.eclipse.january.metadata.internal.ErrorMetadataImpl; 032import org.eclipse.january.metadata.internal.StatisticsMetadataImpl; 033 034/** 035 * Generic container class for data 036 * <p/> 037 * Each subclass has an array of primitive types, elements of this array are grouped or 038 * compounded to make items 039 * <p/> 040 * Data items can be boolean, integer, float, complex float, vector float, etc 041 */ 042public abstract class AbstractDataset extends LazyDatasetBase implements Dataset { 043 // pin UID to base class 044 private static final long serialVersionUID = Dataset.serialVersionUID; 045 046 protected int size; // number of items 047 048 transient protected AbstractDataset base; // is null when not a view 049 protected int[] stride; // can be null for row-major, contiguous datasets 050 protected int offset; 051 052 /** 053 * The data itself, held in a 1D array, but the object will wrap it to appear as possessing as many dimensions as 054 * wanted 055 */ 056 protected Serializable odata = null; 057 058 /** 059 * Set aliased data as base data 060 */ 061 abstract protected void setData(); 062 063 /** 064 * Constructor required for serialisation. 065 */ 066 public AbstractDataset() { 067 } 068 069 @Override 070 public synchronized Dataset synchronizedCopy() { 071 return clone(); 072 } 073 074 @Override 075 public boolean equals(Object obj) { 076 if (this == obj) { 077 return true; 078 } 079 if (obj == null) { 080 return false; 081 } 082 if (!getClass().equals(obj.getClass())) { 083 if (getRank() == 0) // for zero-rank datasets 084 return obj.equals(getObjectAbs(0)); 085 return false; 086 } 087 088 Dataset other = (Dataset) obj; 089 if (getElementsPerItem() != other.getElementsPerItem()) 090 return false; 091 if (size != other.getSize()) 092 return false; 093 if (!Arrays.equals(shape, other.getShapeRef())) { 094 return false; 095 } 096 097 return true; 098 } 099 100 @Override 101 public int hashCode() { 102 return getStats().getHash(shape); 103 } 104 105 @Override 106 abstract public AbstractDataset clone(); 107 108 protected Format stringFormat = null; 109 110 @Override 111 public void setStringFormat(Format format) { 112 stringFormat = format; 113 } 114 115 @Override 116 public Dataset copy(final int dtype) { 117 if (getDType() == dtype) { 118 return clone(); 119 } 120 return DatasetUtils.copy(this, dtype); 121 } 122 123 @Override 124 public <T extends Dataset> T copy(Class<T> clazz) { 125 return DatasetUtils.copy(clazz, this); 126 } 127 128 @Override 129 public Dataset cast(final int dtype) { 130 if (getDType() == dtype) { 131 return this; 132 } 133 return DatasetUtils.cast(this, dtype); 134 } 135 136 @Override 137 public <T extends Dataset> T cast(Class<T> clazz) { 138 return DatasetUtils.cast(clazz, this); 139 } 140 141 @Override 142 public Dataset cast(final boolean repeat, final int dtype, final int isize) { 143 if (getDType() == dtype && getElementsPerItem() == isize) { 144 return this; 145 } 146 return DatasetUtils.cast(this, repeat, dtype, isize); 147 } 148 149 @Override 150 abstract public AbstractDataset getView(boolean deepCopyMetadata); 151 152 /** 153 * Copy fields from original to view 154 * @param orig 155 * @param view 156 * @param clone if true, then clone everything but bulk data 157 * @param cloneMetadata if true, clone metadata 158 */ 159 protected static void copyToView(Dataset orig, AbstractDataset view, boolean clone, boolean cloneMetadata) { 160 view.name = orig.getName(); 161 view.size = orig.getSize(); 162 view.odata = orig.getBuffer(); 163 view.offset = orig.getOffset(); 164 view.base = orig instanceof AbstractDataset ? ((AbstractDataset) orig).base : null; 165 166 if (clone) { 167 view.shape = orig.getShape(); 168 view.stride = orig instanceof AbstractDataset && ((AbstractDataset) orig).stride != null ? 169 ((AbstractDataset) orig).stride.clone() : null; 170 } else { 171 view.shape = orig.getShapeRef(); 172 view.stride = orig instanceof AbstractDataset ? ((AbstractDataset) orig).stride : null; 173 } 174 175 view.metadata = getMetadataMap(orig, cloneMetadata); 176 int odtype = orig.getDType(); 177 int vdtype = view.getDType(); 178 if (odtype != vdtype) { 179 view.setDirty(); 180 } 181 } 182 183 /** 184 * @since 2.0 185 */ 186 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> getMetadataMap(Dataset a, boolean clone) { 187 if (a == null) 188 return null; 189 190 List<MetadataType> all = null; 191 try { 192 all = a.getMetadata(null); 193 } catch (Exception e) { 194 } 195 if (all == null) 196 return null; 197 198 ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 199 200 for (MetadataType m : all) { 201 if (m == null) { 202 continue; 203 } 204 Class<? extends MetadataType> c = findMetadataTypeSubInterfaces(m.getClass()); 205 List<MetadataType> l = map.get(c); 206 if (l == null) { 207 l = new ArrayList<MetadataType>(); 208 map.put(c, l); 209 } 210 if (clone) 211 m = m.clone(); 212 l.add(m); 213 } 214 return map; 215 } 216 217 @Override 218 public IntegerDataset getIndices() { 219 final IntegerDataset ret = DatasetUtils.indices(shape); 220 if (getName() != null) { 221 ret.setName("Indices of " + getName()); 222 } 223 return ret; 224 } 225 226 @Override 227 public Dataset getTransposedView(int... axes) { 228 axes = checkPermutatedAxes(shape, axes); 229 230 AbstractDataset t = getView(true); 231 if (axes == null || getRank() == 1) 232 return t; 233 234 int rank = shape.length; 235 int[] tstride = new int[rank]; 236 int[] toffset = new int[1]; 237 int[] nshape = createStrides(new SliceND(shape), this, tstride, toffset); 238 int[] nstride = new int[rank]; 239 for (int i = 0; i < rank; i++) { 240 final int ax = axes[i]; 241 nstride[i] = tstride[ax]; 242 nshape[i] = shape[ax]; 243 } 244 t.shape = nshape; 245 t.stride = nstride; 246 t.offset = toffset[0]; 247 t.base = this; 248 t.setDirty(); 249 t.transposeMetadata(axes); 250 return t; 251 } 252 253 @Override 254 public Dataset transpose(int... axes) { 255 Dataset t = getTransposedView(axes); 256 return t == null ? clone() : t.clone(); 257 } 258 259 @Override 260 public Dataset swapAxes(int axis1, int axis2) { 261 int rank = shape.length; 262 if (axis1 < 0) 263 axis1 += rank; 264 if (axis2 < 0) 265 axis2 += rank; 266 267 if (axis1 < 0 || axis2 < 0 || axis1 >= rank || axis2 >= rank) { 268 logger.error("Axis value invalid - out of range"); 269 throw new IllegalArgumentException("Axis value invalid - out of range"); 270 } 271 272 if (rank == 1 || axis1 == axis2) { 273 return this; 274 } 275 276 int[] axes = new int[rank]; 277 for (int i = 0; i < rank; i++) { 278 axes[i] = i; 279 } 280 281 axes[axis1] = axis2; 282 axes[axis2] = axis1; 283 return getTransposedView(axes); 284 } 285 286 private boolean isContiguous() { 287 if (stride == null) 288 return true; 289 290 if (offset != 0) 291 return false; 292 293 int s = getElementsPerItem(); 294 for (int j = getRank() - 1; j >= 0; j--) { 295 if (stride[j] != s) { 296 return false; 297 } 298 s *= shape[j]; 299 } 300 301 return true; 302 } 303 304 @Override 305 public Dataset flatten() { 306 if (!isContiguous()) { // need to make a copy if not contiguous 307 return clone().flatten(); 308 } 309 return reshape(size); 310 } 311 312 /** 313 * Fill dataset from object at depth dimension 314 * @param obj 315 * @param depth 316 * @param pos position 317 */ 318 protected void fillData(Object obj, final int depth, final int[] pos) { 319 if (obj == null) { 320 int dtype = getDType(); 321 if (dtype == FLOAT32) 322 set(Float.NaN, pos); 323 else if (dtype == FLOAT64) 324 set(Double.NaN, pos); 325 return; 326 } 327 328 Class<?> clazz = obj.getClass(); 329 if (obj instanceof List<?>) { 330 List<?> jl = (List<?>) obj; 331 int l = jl.size(); 332 for (int i = 0; i < l; i++) { 333 Object lo = jl.get(i); 334 fillData(lo, depth + 1, pos); 335 pos[depth]++; 336 } 337 pos[depth] = 0; 338 } else if (clazz.isArray()) { 339 int l = Array.getLength(obj); 340 if (clazz.equals(odata.getClass())) { 341 System.arraycopy(obj, 0, odata, get1DIndex(pos), l); 342 } else if (clazz.getComponentType().isPrimitive()) { 343 for (int i = 0; i < l; i++) { 344 set(Array.get(obj, i), pos); 345 pos[depth]++; 346 } 347 pos[depth] = 0; 348 } else { 349 for (int i = 0; i < l; i++) { 350 fillData(Array.get(obj, i), depth + 1, pos); 351 pos[depth]++; 352 } 353 pos[depth] = 0; 354 } 355 } else if (obj instanceof IDataset) { 356 boolean[] a = new boolean[shape.length]; 357 for (int i = depth; i < a.length; i++) 358 a[i] = true; 359 setSlice(obj, getSliceIteratorFromAxes(pos, a)); 360 } else { 361 set(obj, pos); 362 } 363 } 364 365 @Override 366 public IndexIterator getIterator(final boolean withPosition) { 367 if (stride != null) { 368 return base.getSize() == 1 ? (withPosition ? new PositionIterator(offset, shape) : 369 new SingleItemIterator(offset, size)) : new StrideIterator(shape, stride, offset); 370 } 371 return withPosition ? new ContiguousIteratorWithPosition(shape, size) : new ContiguousIterator(size); 372 } 373 374 @Override 375 public IndexIterator getIterator() { 376 return getIterator(false); 377 } 378 379 @Override 380 public PositionIterator getPositionIterator(final int... axes) { 381 return new PositionIterator(shape, axes); 382 } 383 384 @Override 385 public IndexIterator getSliceIterator(final int[] start, final int[] stop, final int[] step) { 386 return getSliceIterator(new SliceND(shape, start, stop, step)); 387 } 388 389 /** 390 * @param slice 391 * @return an slice iterator that operates like an IndexIterator 392 */ 393 public IndexIterator getSliceIterator(SliceND slice) { 394 if (ShapeUtils.calcLongSize(slice.getShape()) == 0) { 395 return new NullIterator(shape, slice.getShape()); 396 } 397 if (stride != null) 398 return new StrideIterator(getElementsPerItem(), shape, stride, offset, slice); 399 400 return new SliceIterator(shape, size, slice); 401 } 402 403 @Override 404 public SliceIterator getSliceIteratorFromAxes(final int[] pos, boolean[] axes) { 405 int rank = shape.length; 406 int[] start; 407 int[] stop = new int[rank]; 408 int[] step = new int[rank]; 409 410 if (pos == null) { 411 start = new int[rank]; 412 } else if (pos.length == rank) { 413 start = pos.clone(); 414 } else { 415 throw new IllegalArgumentException("pos array length is not equal to rank of dataset"); 416 } 417 if (axes == null) { 418 axes = new boolean[rank]; 419 Arrays.fill(axes, true); 420 } else if (axes.length != rank) { 421 throw new IllegalArgumentException("axes array length is not equal to rank of dataset"); 422 } 423 424 for (int i = 0; i < rank; i++) { 425 if (axes[i]) { 426 stop[i] = shape[i]; 427 } else { 428 stop[i] = start[i] + 1; 429 } 430 step[i] = 1; 431 } 432 return (SliceIterator) getSliceIterator(start, stop, step); 433 } 434 435 @Override 436 public BooleanIterator getBooleanIterator(Dataset choice) { 437 return getBooleanIterator(choice, true); 438 } 439 440 @Override 441 public BooleanIterator getBooleanIterator(Dataset choice, boolean value) { 442 return new BooleanIterator(getIterator(), choice, value); 443 } 444 445 @Override 446 public Dataset getByBoolean(Dataset selection) { 447 checkCompatibility(selection); 448 449 final int length = ((Number) selection.sum()).intValue(); 450 final int is = getElementsPerItem(); 451 @SuppressWarnings("deprecation") 452 Dataset r = DatasetFactory.zeros(is, new int[] { length }, getDType()); 453 BooleanIterator biter = getBooleanIterator(selection); 454 455 int i = 0; 456 while (biter.hasNext()) { 457 r.setObjectAbs(i, getObjectAbs(biter.index)); 458 i += is; 459 } 460 return r; 461 } 462 463 @Override 464 public Dataset getBy1DIndex(IntegerDataset index) { 465 final int is = getElementsPerItem(); 466 @SuppressWarnings("deprecation") 467 final Dataset r = DatasetFactory.zeros(is, index.getShape(), getDType()); 468 final IntegerIterator iter = new IntegerIterator(index, size, is); 469 470 int i = 0; 471 while (iter.hasNext()) { 472 r.setObjectAbs(i, getObjectAbs(iter.index)); 473 i += is; 474 } 475 return r; 476 } 477 478 @Override 479 public Dataset getByIndexes(final Object... indexes) { 480 final IntegersIterator iter = new IntegersIterator(shape, indexes); 481 final int is = getElementsPerItem(); 482 @SuppressWarnings("deprecation") 483 final Dataset r = DatasetFactory.zeros(is, iter.getShape(), getDType()); 484 485 final int[] pos = iter.getPos(); 486 int i = 0; 487 while (iter.hasNext()) { 488 r.setObjectAbs(i, getObject(pos)); 489 i += is; 490 } 491 return r; 492 } 493 494 @Override 495 public Class<?> getElementClass() { 496 return DTypeUtils.getElementClass(getDType()); 497 } 498 499 @Override 500 public boolean hasFloatingPointElements() { 501 Class<?> cls = getElementClass(); 502 return cls == Float.class || cls == Double.class; 503 } 504 505 @Override 506 public int getElementsPerItem() { 507 return DTypeUtils.getElementsPerItem(getDType()); 508 } 509 510 @Override 511 public int getItemBytes() { 512 return DTypeUtils.getItemBytes(getDType(), getElementsPerItem()); 513 } 514 515 @Override 516 public String getName() { 517 return name; 518 } 519 520 @Override 521 public void setName(final String name) { 522 this.name = name; 523 } 524 525 @Override 526 public int getSize() { 527 return size; 528 } 529 530 @Override 531 public int[] getShape() { 532 // make a copy of the dimensions data, and put that out 533 if (shape == null) { 534 logger.warn("Shape is null!!!"); 535 return new int[] {}; 536 } 537 return shape.clone(); 538 } 539 540 @Override 541 public int getRank() { 542 return shape == null ? 0 : shape.length; 543 } 544 545 @Override 546 public int getNbytes() { 547 return getSize() * getItemBytes(); 548 } 549 550 /** 551 * Check for -1 placeholder in shape and replace if necessary 552 * @param shape 553 * @param size 554 */ 555 private void checkShape(int[] shape, int size) { 556 int rank = shape.length; 557 int found = -1; 558 int nsize = 1; 559 for (int i = 0; i < rank; i++) { 560 int d = shape[i]; 561 if (d == -1) { 562 if (found == -1) { 563 found = i; 564 } else { 565 logger.error("Can only have one -1 placeholder in shape"); 566 throw new IllegalArgumentException("Can only have one -1 placeholder in shape"); 567 } 568 } else { 569 nsize *= d; 570 } 571 } 572 if (found >= 0) { 573 shape[found] = size/nsize; 574 } else if (nsize != size && !(rank == 0 && size == 0)) { 575 logger.error("New shape is not same size as old shape"); 576 throw new IllegalArgumentException("New size is not same as the old size. Old size is "+size+" new size is "+nsize+" and shape is "+Arrays.toString(shape)); 577 } 578 } 579 580 @Override 581 public void setShape(final int... shape) { 582 int[] nshape = shape.clone(); 583 checkShape(nshape, size); 584 if (Arrays.equals(this.shape, nshape)) { 585 return; 586 } 587 588 if (stride != null) { 589 // the only compatible shapes are ones where new dimensions are factors of old dimensions 590 // or are combined adjacent old dimensions 591 int[] oshape = this.shape; 592 int orank = oshape.length; 593 int nrank = nshape.length; 594 int diff = nrank - orank; 595 int[] nstride = new int[nrank]; 596 boolean ones = true; 597 // work forwards for broadcasting cases 598 for (int i = 0, j = 0; i < orank || j < nrank;) { 599 if (j >= diff && i < orank && j < nrank && oshape[i] == nshape[j]) { 600 nstride[j++] = stride[i++]; 601 } else if (j < nrank && nshape[j] == 1) { 602 nstride[j++] = 0; 603 } else if (i < orank && oshape[i] == 1) { 604 i++; 605 } else { 606 if (j < nrank) 607 j++; 608 if (i < orank) 609 i++; 610 ones = false; 611 } 612 } 613 if (!ones) { // not just ones differ in shapes 614 int[] ostride = stride; 615 int ob = 0; 616 int oe = 1; 617 int nb = 0; 618 int ne = 1; 619 while (ob < orank && nb < nrank) { 620 int ol = oshape[ob]; 621 int nl = nshape[nb]; 622 623 if (nl < ol) { // find group of shape dimensions that form common size 624 do { // case where new shape spreads single dimension over several dimensions 625 if (ne == nrank) { 626 break; 627 } 628 nl *= nshape[ne++]; 629 } while (nl < ol); 630 if (nl != ol) { 631 logger.error("Subshape is incompatible with single dimension"); 632 throw new IllegalArgumentException("Subshape is incompatible with single dimension"); 633 } 634 int on = ne - 1; 635 while (nshape[on] == 1) { 636 on--; 637 } 638 639 nstride[on] = ostride[ob]; 640 for (int n = on - 1; n >= nb; n--) { 641 if (nshape[n] == 1) 642 continue; 643 644 nstride[n] = nshape[on] * nstride[on]; 645 on = n; 646 } 647 } else if (ol < nl) { 648 do { // case where new shape combines several dimensions into one dimension 649 if (oe == orank) { 650 break; 651 } 652 ol *= oshape[oe++]; 653 } while (ol < nl); 654 if (nl != ol) { 655 logger.error("Single dimension is incompatible with subshape"); 656 throw new IllegalArgumentException("Single dimension is incompatible with subshape"); 657 } 658 659 int oo = oe - 1; 660 while (oshape[oo] == 1) { 661 oo--; 662 } 663 int os = ostride[oo]; 664 for (int o = oo - 1; o >= ob; o--) { 665 if (oshape[o] == 1) 666 continue; 667 if (ostride[o] != oshape[oo] * ostride[oo]) { 668 logger.error("Subshape cannot be a non-contiguous view"); 669 throw new IllegalArgumentException("Subshape cannot be a non-contiguous view"); 670 } 671 oo = o; 672 } 673 nstride[nb] = os; 674 } else { 675 nstride[nb] = ostride[ob]; 676 } 677 678 ob = oe++; 679 nb = ne++; 680 } 681 } 682 683 stride = nstride; 684 } 685 686 setDirty(); 687 if (this.shape != null) { 688 reshapeMetadata(this.shape, nshape); 689 this.shape = nshape; 690 } 691 } 692 693 @Override 694 public int[] getShapeRef() { 695 return shape; 696 } 697 698 @Override 699 public int getOffset() { 700 return offset; 701 } 702 703 @Override 704 public int[] getStrides() { 705 return stride; 706 } 707 708 @Override 709 public Serializable getBuffer() { 710 return odata; 711 } 712 713 @Override 714 public void overrideInternal(Serializable buffer, int... shape) { 715 if (buffer != null) { 716 odata = buffer; 717 setData(); 718 setDirty(); 719 } 720 721 if (shape != null) { 722 this.shape = shape.clone(); 723 size = ShapeUtils.calcSize(this.shape); 724 } 725 } 726 727 /** 728 * Create a stride array from dataset 729 * @param a dataset 730 * @param offset output offset 731 * @return new strides 732 */ 733 public static int[] createStrides(Dataset a, final int[] offset) { 734 return createStrides(a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), offset); 735 } 736 737 /** 738 * Create a stride array from dataset 739 * @param isize 740 * @param shape 741 * @param oStride original stride 742 * @param oOffset original offset (only used if there is an original stride) 743 * @param offset output offset 744 * @return new strides 745 */ 746 public static int[] createStrides(final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] offset) { 747 int rank = shape.length; 748 final int[] stride; 749 if (oStride == null) { 750 offset[0] = 0; 751 stride = new int[rank]; 752 int s = isize; 753 for (int j = rank - 1; j >= 0; j--) { 754 stride[j] = s; 755 s *= shape[j]; 756 } 757 } else { 758 offset[0] = oOffset; 759 stride = oStride.clone(); 760 } 761 return stride; 762 } 763 764 /** 765 * Create a stride array from slice information and a dataset 766 * @param slice 767 * @param a dataset 768 * @param stride output stride 769 * @param offset output offset 770 * @return new shape 771 */ 772 public static int[] createStrides(final SliceND slice, final Dataset a, final int[] stride, final int[] offset) { 773 return createStrides(slice, a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), stride, offset); 774 } 775 776 /** 777 * Create a stride array from slice and dataset information 778 * @param slice 779 * @param isize 780 * @param shape 781 * @param oStride original stride 782 * @param oOffset original offset (only used if there is an original stride) 783 * @param stride output stride 784 * @param offset output offset 785 * @return new shape 786 */ 787 public static int[] createStrides(final SliceND slice, final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] stride, final int[] offset) { 788 int[] lstart = slice.getStart(); 789 int[] lstep = slice.getStep(); 790 int[] newShape = slice.getShape(); 791 int rank = shape.length; 792 793 if (oStride == null) { 794 int s = isize; 795 offset[0] = 0; 796 for (int j = rank - 1; j >= 0; j--) { 797 stride[j] = s * lstep[j]; 798 offset[0] += s * lstart[j]; 799 s *= shape[j]; 800 } 801 } else { 802 offset[0] = oOffset; 803 for (int j = 0; j < rank; j++) { 804 int s = oStride[j]; 805 stride[j] = lstep[j] * s; 806 offset[0] += lstart[j] * s; 807 } 808 } 809 810 return newShape; 811 } 812 813 @Override 814 public Dataset getBroadcastView(int... broadcastShape) { 815 AbstractDataset view = getView(true); 816 817 if (!Arrays.equals(shape, broadcastShape)) { 818 List<int[]> nShapes = BroadcastUtils.broadcastShapesToMax(broadcastShape, shape); 819 view.setShape(nShapes.get(0)); 820 view.stride = BroadcastUtils.createBroadcastStrides(view, broadcastShape); 821 view.base = this; 822 view.shape = broadcastShape.clone(); 823 view.size = ShapeUtils.calcSize(broadcastShape); 824 if (view.name == null || view.name.isEmpty()) { 825 view.name = "Broadcast from " + Arrays.toString(shape); 826 } else { 827 view.name = "Broadcast of " + view.name + " from " + Arrays.toString(shape); 828 } 829 } 830 return view; 831 } 832 833 @Override 834 public Dataset getSliceView(final int[] start, final int[] stop, final int[] step) { 835 return getSliceView(new SliceND(shape, start, stop, step)); 836 } 837 838 @Override 839 public Dataset getSliceView(Slice... slice) { 840 if (slice == null || slice.length == 0) { 841 return getView(true); 842 } 843 844 return getSliceView(new SliceND(shape, slice)); 845 } 846 847 /** 848 * Get a slice of the dataset. The returned dataset is a view on a selection of items 849 * @param slice 850 * @return slice view 851 */ 852 @Override 853 public Dataset getSliceView(SliceND slice) { 854 if (slice.isAll()) { 855 return getView(true); 856 } 857 858 final int rank = shape.length; 859 int[] sStride = new int[rank]; 860 int[] sOffset = new int[1]; 861 862 int[] sShape = createStrides(slice, this, sStride, sOffset); 863 864 AbstractDataset s = getView(false); 865 s.shape = sShape; 866 s.size = ShapeUtils.calcSize(sShape); 867 s.stride = sStride; 868 s.offset = sOffset[0]; 869 s.base = this; 870 871 s.metadata = copyMetadata(); 872 s.sliceMetadata(true, slice); 873 874 s.setDirty(); 875 s.setName(name + BLOCK_OPEN + slice + BLOCK_CLOSE); 876 877 return s; 878 } 879 880 /** 881 * Get flattened view index of given position 882 * @param pos 883 * the integer array specifying the n-D position 884 * @return the index on the flattened dataset 885 */ 886 private int getFlat1DIndex(final int[] pos) { 887 final int imax = pos.length; 888 if (imax == 0) { 889 return 0; 890 } 891 892 return get1DIndexFromShape(pos); 893 } 894 895 /** 896 * @since 2.0 897 */ 898 protected int getFirst1DIndex() { 899 if (shape == null) { 900 throw new IllegalArgumentException("Cannot find an index from a null shape"); 901 } 902 return stride == null ? 0 : offset; 903 } 904 905 @Override 906 public int get1DIndex(final int... n) { 907 if (n.length == 0 && shape.length == 0) 908 return offset; 909 910 return stride == null ? get1DIndexFromShape(n) : get1DIndexFromStrides(n); 911 } 912 913 private static void throwAIOOBException(int i, int s, int d) { 914 throw new ArrayIndexOutOfBoundsException("Index (" + i + ") out of range [-" + s + "," + s 915 + "] in dimension " + d); 916 } 917 918 /** 919 * @param i 920 * @return the index on the data array corresponding to that location 921 */ 922 protected int get1DIndex(int i) { 923 if (shape == null) { 924 throw new IllegalArgumentException("Cannot find an index from a null shape"); 925 } 926 if (shape.length > 1) { 927 logger.error("This dataset is not 1D but was addressed as such"); 928 throw new UnsupportedOperationException("This dataset is not 1D but was addressed as such"); 929 } 930 if (i < 0) { 931 i += shape[0]; 932 } 933 if (i < 0 || i >= shape[0]) { 934 throwAIOOBException(i, shape[0], 0); 935 } 936 return stride == null ? i : i*stride[0] + offset; 937 } 938 939 /** 940 * @param i 941 * @param j 942 * @return the index on the data array corresponding to that location 943 */ 944 protected int get1DIndex(int i, int j) { 945 if (shape == null) { 946 throw new IllegalArgumentException("Cannot find an index from a null shape"); 947 } 948 if (shape.length != 2) { 949 logger.error("This dataset is not 2D but was addressed as such"); 950 throw new UnsupportedOperationException("This dataset is not 2D but was addressed as such"); 951 } 952 if (i < 0) { 953 i += shape[0]; 954 } 955 if (i < 0 || i >= shape[0]) { 956 throwAIOOBException(i, shape[0], 0); 957 } 958 if (j < 0) { 959 j += shape[1]; 960 } 961 if (j < 0 || j >= shape[1]) { 962 throwAIOOBException(i, shape[1], 1); 963 } 964 return stride == null ? i*shape[1] + j : i*stride[0] + j*stride[1] + offset; 965 } 966 967 protected int get1DIndexFromShape(final int[] n) { 968 return get1DIndexFromShape(shape, n); 969 } 970 971 protected static int get1DIndexFromShape(final int[] shape, final int[] n) { 972 if (shape == null) { 973 throw new IllegalArgumentException("Cannot find an index from a null shape"); 974 } 975 final int rank = shape.length; 976 if (rank != n.length) { 977 String errMsg = String.format("Number of position values must be equal to rank of %d", rank); 978 logger.error(errMsg); 979 throw new IllegalArgumentException(errMsg); 980 } 981 int index = 0; 982 for (int i = 0; i < rank; i++) { 983 final int si = shape[i]; 984 int ni = n[i]; 985 if (ni < 0) { 986 ni += si; 987 } 988 if (ni < 0 || ni >= si) { 989 throwAIOOBException(ni, si, i); 990 } 991 index = index * si + ni; 992 } 993 994 return index; 995 } 996 997 private int get1DIndexFromStrides(final int[] n) { 998 return get1DIndexFromStrides(shape, stride, offset, n); 999 } 1000 1001 private static int get1DIndexFromStrides(final int[] shape, final int[] stride, final int offset, final int[] n) { 1002 if (shape == null) { 1003 throw new IllegalArgumentException("Cannot find an index from a null shape"); 1004 } 1005 final int rank = shape.length; 1006 if (rank != n.length) { 1007 String errMsg = String.format("Number of position values must be equal to rank of %d", rank); 1008 logger.error(errMsg); 1009 throw new IllegalArgumentException(errMsg); 1010 } 1011 int index = offset; 1012 for (int i = 0; i < rank; i++) { 1013 final int st = stride[i]; 1014 if (st != 0) { // not broadcasted 1015 final int si = shape[i]; 1016 int ni = n[i]; 1017 if (ni < 0) { 1018 ni += si; 1019 } 1020 if (ni < 0 || ni >= si) { 1021 throwAIOOBException(ni, si, i); 1022 } 1023 index += st * ni; 1024 } 1025 } 1026 return index; 1027 } 1028 1029 @Override 1030 public int[] getNDPosition(final int n) { 1031 if (isIndexInRange(n)) { 1032 throw new IllegalArgumentException("Index provided " + n 1033 + "is larger then the size of the containing array"); 1034 } 1035 1036 return stride == null ? ShapeUtils.getNDPositionFromShape(n, shape) : getNDPositionFromStrides(n); 1037 } 1038 1039 private boolean isIndexInRange(final int n) { 1040 if (stride == null) { 1041 return n >= size; 1042 } 1043 return n >= getBufferLength(); 1044 } 1045 1046 /** 1047 * @return entire buffer length 1048 */ 1049 abstract protected int getBufferLength(); 1050 1051 private int[] getNDPositionFromStrides(int n) { 1052 n -= offset; 1053 int rank = shape.length; 1054 if (rank == 1) { 1055 return new int[] { n / stride[0] }; 1056 } 1057 1058 int[] output = new int[rank]; 1059 int i = 0; 1060 while (i != n) { // TODO find more efficient way than this exhaustive search 1061 int j = rank - 1; 1062 for (; j >= 0; j--) { 1063 output[j]++; 1064 i += stride[j]; 1065 if (output[j] >= shape[j]) { 1066 output[j] = 0; 1067 i -= shape[j] * stride[j]; 1068 } else { 1069 break; 1070 } 1071 } 1072 if (j == -1) { 1073 logger.error("Index was not found in this strided dataset"); 1074 throw new IllegalArgumentException("Index was not found in this strided dataset"); 1075 } 1076 } 1077 1078 return output; 1079 } 1080 1081 @Override 1082 public int checkAxis(int axis) { 1083 return checkAxis(shape.length, axis); 1084 } 1085 1086 /** 1087 * Check that axis is in range [-rank,rank) 1088 * 1089 * @param rank 1090 * @param axis 1091 * @return sanitized axis in range [0, rank) 1092 */ 1093 protected static int checkAxis(int rank, int axis) { 1094 if (axis < 0) { 1095 axis += rank; 1096 } 1097 1098 if (axis < 0 || axis >= rank) { 1099 throw new IndexOutOfBoundsException("Axis " + axis + " given is out of range [0, " + rank + ")"); 1100 } 1101 return axis; 1102 } 1103 1104 protected static final char BLOCK_OPEN = '['; 1105 protected static final char BLOCK_CLOSE = ']'; 1106 1107 @Override 1108 public String toString() { 1109 return toString(false); 1110 } 1111 1112 @Override 1113 public String toString(boolean showData) { 1114 final int rank = shape == null ? 0 : shape.length; 1115 final StringBuilder out = new StringBuilder(); 1116 if (DTypeUtils.isDTypeElemental(getDType())) { 1117 out.append("Dataset "); 1118 } else { 1119 out.append("Compound dataset ("); 1120 out.append(getElementsPerItem()); 1121 out.append(") "); 1122 } 1123 1124 if (!showData) { 1125 if (name != null && name.length() > 0) { 1126 out.append("'"); 1127 out.append(name); 1128 out.append("' has shape "); 1129 } else { 1130 out.append("shape is "); 1131 } 1132 1133 out.append(BLOCK_OPEN); 1134 if (rank > 0) { 1135 out.append(shape[0]); 1136 } 1137 for (int i = 1; i < rank; i++) { 1138 out.append(", " + shape[i]); 1139 } 1140 out.append(BLOCK_CLOSE); 1141 return out.toString(); 1142 } 1143 1144 if (size == 0) { 1145 return out.toString(); 1146 } 1147 1148 if (rank > 0) { 1149 int[] pos = new int[rank]; 1150 final StringBuilder lead = new StringBuilder(); 1151 printBlocks(out, lead, 0, pos); 1152 } else { 1153 out.append(getString()); 1154 } 1155 return out.toString(); 1156 } 1157 1158 /** 1159 * Limit to strings output via the toString() method 1160 */ 1161 private static int maxStringLength = 120; 1162 1163 /** 1164 * Set maximum line length for toString() method 1165 * @param maxLineLength 1166 */ 1167 public static void setMaxLineLength(int maxLineLength) { 1168 maxStringLength = maxLineLength; 1169 } 1170 1171 /** 1172 * @return maximum line length for toString() method 1173 */ 1174 public static int getMaxLineLength() { 1175 return maxStringLength; 1176 } 1177 1178 /** 1179 * Limit to number of sub-blocks output via the toString() method 1180 */ 1181 private static final int MAX_SUBBLOCKS = 6; 1182 1183 private final static String SEPARATOR = ","; 1184 private final static String SPACE = " "; 1185 private final static String ELLIPSIS = "..."; 1186 private final static String NEWLINE = "\n"; 1187 1188 /** 1189 * Make a line of output for last dimension of dataset 1190 * 1191 * @param start 1192 * @return line 1193 */ 1194 private StringBuilder makeLine(final int end, final int... start) { 1195 StringBuilder line = new StringBuilder(); 1196 final int[] pos; 1197 if (end >= start.length) { 1198 pos = Arrays.copyOf(start, end + 1); 1199 } else { 1200 pos = start; 1201 } 1202 pos[end] = 0; 1203 line.append(BLOCK_OPEN); 1204 line.append(getString(pos)); 1205 1206 final int length = shape[end]; 1207 1208 // trim elements printed if length exceed estimate of maximum elements 1209 int excess = length - maxStringLength / 3; // space + number + separator 1210 if (excess > 0) { 1211 int index = (length - excess) / 2; 1212 for (int y = 1; y < index; y++) { 1213 line.append(SEPARATOR + SPACE); 1214 pos[end] = y; 1215 line.append(getString(pos)); 1216 } 1217 index = (length + excess) / 2; 1218 for (int y = index; y < length; y++) { 1219 line.append(SEPARATOR + SPACE); 1220 pos[end] = y; 1221 line.append(getString(pos)); 1222 } 1223 } else { 1224 for (int y = 1; y < length; y++) { 1225 line.append(SEPARATOR + SPACE); 1226 pos[end] = y; 1227 line.append(getString(pos)); 1228 } 1229 } 1230 line.append(BLOCK_CLOSE); 1231 1232 // trim string down to limit 1233 excess = line.length() - maxStringLength - ELLIPSIS.length() - 1; 1234 if (excess > 0) { 1235 int index = line.substring(0, (line.length() - excess) / 2).lastIndexOf(SEPARATOR) + 2; 1236 StringBuilder out = new StringBuilder(line.subSequence(0, index)); 1237 out.append(ELLIPSIS + SEPARATOR); 1238 index = line.substring((line.length() + excess) / 2).indexOf(SEPARATOR) + (line.length() + excess) / 2 + 1; 1239 out.append(line.subSequence(index, line.length())); 1240 return out; 1241 } 1242 1243 return line; 1244 } 1245 1246 /** 1247 * recursive method to print blocks 1248 */ 1249 private void printBlocks(final StringBuilder out, final StringBuilder lead, final int level, final int[] pos) { 1250 if (out.length() > 0) { 1251 char last = out.charAt(out.length() - 1); 1252 if (last != BLOCK_OPEN) { 1253 out.append(lead); 1254 } 1255 } 1256 final int end = getRank() - 1; 1257 if (level != end) { 1258 out.append(BLOCK_OPEN); 1259 int length = shape[level]; 1260 1261 // first sub-block 1262 pos[level] = 0; 1263 StringBuilder newlead = new StringBuilder(lead); 1264 newlead.append(SPACE); 1265 printBlocks(out, newlead, level + 1, pos); 1266 if (length < 2) { // escape 1267 out.append(BLOCK_CLOSE); 1268 return; 1269 } 1270 1271 out.append(SEPARATOR + NEWLINE); 1272 for (int i = level + 1; i < end; i++) { 1273 out.append(NEWLINE); 1274 } 1275 1276 // middle sub-blocks 1277 if (length < MAX_SUBBLOCKS) { 1278 for (int x = 1; x < length - 1; x++) { 1279 pos[level] = x; 1280 printBlocks(out, newlead, level + 1, pos); 1281 if (end <= level + 1) { 1282 out.append(SEPARATOR + NEWLINE); 1283 } else { 1284 out.append(SEPARATOR + NEWLINE + NEWLINE); 1285 } 1286 } 1287 } else { 1288 final int excess = length - MAX_SUBBLOCKS; 1289 int xmax = (length - excess) / 2; 1290 for (int x = 1; x < xmax; x++) { 1291 pos[level] = x; 1292 printBlocks(out, newlead, level + 1, pos); 1293 if (end <= level + 1) { 1294 out.append(SEPARATOR + NEWLINE); 1295 } else { 1296 out.append(SEPARATOR + NEWLINE + NEWLINE); 1297 } 1298 } 1299 out.append(newlead); 1300 out.append(ELLIPSIS + SEPARATOR + NEWLINE); 1301 xmax = (length + excess) / 2; 1302 for (int x = xmax; x < length - 1; x++) { 1303 pos[level] = x; 1304 printBlocks(out, newlead, level + 1, pos); 1305 if (end <= level + 1) { 1306 out.append(SEPARATOR + NEWLINE); 1307 } else { 1308 out.append(SEPARATOR + NEWLINE + NEWLINE); 1309 } 1310 } 1311 } 1312 1313 // last sub-block 1314 pos[level] = length - 1; 1315 printBlocks(out, newlead, level + 1, pos); 1316 out.append(BLOCK_CLOSE); 1317 } else { 1318 out.append(makeLine(end, pos)); 1319 } 1320 } 1321 1322 @Override 1323 public Dataset squeezeEnds() { 1324 return squeeze(true); 1325 } 1326 1327 @Override 1328 public Dataset squeeze() { 1329 return squeeze(false); 1330 } 1331 1332 @Override 1333 public Dataset squeeze(boolean onlyFromEnds) { 1334 final int[] tshape = ShapeUtils.squeezeShape(shape, onlyFromEnds); 1335 final int[] oshape = shape; 1336 if (stride == null) { 1337 shape = tshape; 1338 } else { 1339 int rank = shape.length; 1340 int trank = tshape.length; 1341 if (trank < rank) { 1342 int[] tstride = new int[tshape.length]; 1343 if (onlyFromEnds) { 1344 for (int i = 0; i < rank; i++) { 1345 if (shape[i] != 1) { 1346 for (int k = 0; k < trank; k++) { 1347 tstride[k] = stride[i++]; 1348 } 1349 break; 1350 } 1351 } 1352 } else { 1353 int t = 0; 1354 for (int i = 0; i < rank; i++) { 1355 if (shape[i] != 1) { 1356 tstride[t++] = stride[i]; 1357 } 1358 } 1359 } 1360 shape = tshape; 1361 stride = tstride; 1362 } 1363 } 1364 1365 setDirty(); 1366 reshapeMetadata(oshape, shape); 1367 return this; 1368 } 1369 1370 @Override 1371 public boolean isCompatibleWith(final ILazyDataset g) { 1372 return ShapeUtils.areShapesCompatible(shape, g.getShape()); 1373 } 1374 1375 @Override 1376 public void checkCompatibility(final ILazyDataset g) throws IllegalArgumentException { 1377 ShapeUtils.checkCompatibility(this, g); 1378 } 1379 1380 @Override 1381 public Dataset reshape(final int... shape) { 1382 Dataset a; 1383 try { 1384 a = getView(true); 1385 a.setShape(shape); 1386 } catch (IllegalArgumentException e) { 1387 a = clone(); 1388 a.setShape(shape); 1389 } 1390 return a; 1391 } 1392 1393 /** 1394 * @param start 1395 * @param stop 1396 * @param step 1397 * @return number of steps to take 1398 */ 1399 protected static int calcSteps(final double start, final double stop, final double step) { 1400 return Math.max(0, (int) Math.ceil((stop - start) / step)); 1401 } 1402 1403 @Override 1404 public boolean isComplex() { 1405 int type = getDType(); 1406 return type == COMPLEX64 || type == COMPLEX128; 1407 } 1408 1409 @Override 1410 public Dataset getRealPart() { 1411 return this; 1412 } 1413 1414 @Override 1415 public Dataset getRealView() { 1416 return getView(true); 1417 } 1418 1419 @Override 1420 public Dataset getSlice(final int[] start, final int[] stop, final int[] step) { 1421 return getSlice(new SliceND(shape, start, stop, step)); 1422 } 1423 1424 @Override 1425 public Dataset getSlice(Slice... slice) { 1426 return getSlice(new SliceND(shape, slice)); 1427 } 1428 1429 @Override 1430 public Dataset getSlice(IMonitor monitor, Slice... slice) { 1431 return getSlice(slice); 1432 } 1433 1434 @Override 1435 public Dataset getSlice(IMonitor monitor, SliceND slice) { 1436 return getSlice(slice); 1437 } 1438 1439 @Override 1440 public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) { 1441 return getSlice(start, stop, step); 1442 } 1443 1444 /** 1445 * Get a slice of the dataset. The returned dataset is a copied selection of items 1446 * @param slice 1447 * @return The dataset of the sliced data 1448 */ 1449 @Override 1450 public Dataset getSlice(final SliceND slice) { 1451 SliceIterator it = (SliceIterator) getSliceIterator(slice); 1452 AbstractDataset s = getSlice(it); 1453 s.metadata = copyMetadata(); 1454 s.setDirty(); 1455 s.sliceMetadata(true, slice); 1456 return s; 1457 } 1458 1459 /** 1460 * Get a slice of the dataset. The returned dataset is a copied selection of items 1461 * 1462 * @param iterator Slice iterator 1463 * @return The dataset of the sliced data 1464 */ 1465 abstract public AbstractDataset getSlice(final SliceIterator iterator); 1466 1467 @SuppressWarnings("deprecation") 1468 @Override 1469 public Dataset setSlice(final Object obj, final SliceND slice) { 1470 Dataset ds; 1471 if (obj instanceof Dataset) { 1472 ds = (Dataset) obj; 1473 } else if (obj instanceof IDataset) { 1474 ds = DatasetUtils.convertToDataset((IDataset) obj); 1475 } else { 1476 int dtype = getDType(); 1477 if (dtype != Dataset.BOOL) { 1478 dtype = DTypeUtils.getLargestDType(dtype); 1479 } 1480 ds = DatasetFactory.createFromObject(getElementsPerItem(), dtype, obj); 1481 } 1482 1483 return setSlicedView(getSliceView(slice), ds); 1484 } 1485 1486 @Override 1487 public Dataset setSlice(final Object obj, final int[] start, final int[] stop, final int[] step) { 1488 return setSlice(obj, new SliceND(shape, start, stop, step)); 1489 } 1490 1491 /** 1492 * Set a view of current dataset to given dataset with broadcasting 1493 * @param view 1494 * @param d 1495 * @return this dataset 1496 */ 1497 abstract Dataset setSlicedView(Dataset view, Dataset d); 1498 1499 @Override 1500 public Dataset setSlice(Object obj, Slice... slice) { 1501 if (slice == null || slice.length == 0) { 1502 return setSlice(obj, new SliceND(shape)); 1503 } 1504 return setSlice(obj, new SliceND(shape, slice)); 1505 } 1506 1507 @Override 1508 public boolean all() { 1509 return Comparisons.allTrue(this); 1510 } 1511 1512 @Override 1513 public BooleanDataset all(final int axis) { 1514 return Comparisons.allTrue(this, axis); 1515 } 1516 1517 @Override 1518 public boolean any() { 1519 return Comparisons.anyTrue(this); 1520 } 1521 1522 @Override 1523 public BooleanDataset any(final int axis) { 1524 return Comparisons.anyTrue(this, axis); 1525 } 1526 1527 @Override 1528 public Dataset ifloorDivide(final Object o) { 1529 return idivide(o).ifloor(); 1530 } 1531 1532 @Override 1533 public double residual(final Object o) { 1534 return residual(o, null, false); 1535 } 1536 1537 @Override 1538 public double residual(final Object o, boolean ignoreNaNs) { 1539 return residual(o, null, ignoreNaNs); 1540 } 1541 1542 /** 1543 * @since 2.0 1544 */ 1545 @SuppressWarnings("unchecked") 1546 protected StatisticsMetadata<Number> getStats() { 1547 StatisticsMetadata<Number> md = getFirstMetadata(StatisticsMetadata.class); 1548 if (md == null || md.isDirty()) { 1549 md = new StatisticsMetadataImpl<Number>(); 1550 md.initialize(this); 1551 setMetadata(md); 1552 } 1553 return md; 1554 } 1555 1556 /** 1557 * @since 2.0 1558 */ 1559 @SuppressWarnings("unchecked") 1560 protected StatisticsMetadata<String> getStringStats() { 1561 StatisticsMetadata<String> md = getFirstMetadata(StatisticsMetadata.class); 1562 if (md == null || md.isDirty()) { 1563 md = new StatisticsMetadataImpl<String>(); 1564 md.initialize(this); 1565 setMetadata(md); 1566 } 1567 return md; 1568 } 1569 1570 @Override 1571 public Number max(boolean... ignoreInvalids) { 1572 return getStats().getMaximum(ignoreInvalids); 1573 } 1574 1575 @Override 1576 public Dataset max(int axis, boolean... ignoreInvalids) { 1577 return getStats().getMaximum(axis, ignoreInvalids); 1578 } 1579 1580 @Override 1581 public Number min(boolean... ignoreInvalids) { 1582 return getStats().getMinimum(ignoreInvalids); 1583 } 1584 1585 @Override 1586 public Dataset min(int axis, boolean... ignoreInvalids) { 1587 return getStats().getMinimum(axis, ignoreInvalids); 1588 } 1589 1590 @Override 1591 public int argMax(boolean... ignoreInvalids) { 1592 return getFlat1DIndex(maxPos(ignoreInvalids)); 1593 } 1594 1595 /** 1596 * @since 2.0 1597 */ 1598 @Override 1599 public IntegerDataset argMax(int axis, boolean... ignoreInvalids) { 1600 return (IntegerDataset) getStats().getArgMaximum(axis, ignoreInvalids); 1601 } 1602 1603 @Override 1604 public int argMin(boolean... ignoreInvalids) { 1605 return getFlat1DIndex(minPos(ignoreInvalids)); 1606 } 1607 1608 /** 1609 * @since 2.0 1610 */ 1611 @Override 1612 public IntegerDataset argMin(int axis, boolean... ignoreInvalids) { 1613 return (IntegerDataset) getStats().getArgMinimum(axis, ignoreInvalids); 1614 } 1615 1616 @Override 1617 public Number peakToPeak(boolean... ignoreInvalids) { 1618 return DTypeUtils.fromDoubleToBiggestNumber(max(ignoreInvalids).doubleValue() - min(ignoreInvalids).doubleValue(), getDType()); 1619 } 1620 1621 @Override 1622 public Dataset peakToPeak(int axis, boolean... ignoreInvalids) { 1623 return Maths.subtract(max(axis, ignoreInvalids), min(axis, ignoreInvalids)); 1624 } 1625 1626 1627 @Override 1628 public long count(boolean... ignoreInvalids) { 1629 return getStats().getCount(ignoreInvalids); 1630 } 1631 1632 @Override 1633 public Dataset count(int axis, boolean... ignoreInvalids) { 1634 return getStats().getCount(axis, ignoreInvalids); 1635 } 1636 1637 @Override 1638 public Object sum(boolean... ignoreInvalids) { 1639 return getStats().getSum(ignoreInvalids); 1640 } 1641 1642 @Override 1643 public Dataset sum(int axis, boolean... ignoreInvalids) { 1644 return getStats().getSum(axis, ignoreInvalids); 1645 } 1646 1647 @Override 1648 public Object product(boolean... ignoreInvalids) { 1649 return Stats.product(this, ignoreInvalids); 1650 } 1651 1652 @Override 1653 public Dataset product(int axis, boolean... ignoreInvalids) { 1654 return Stats.product(this, axis, ignoreInvalids); 1655 } 1656 1657 @Override 1658 public Object mean(boolean... ignoreInvalids) { 1659 return getStats().getMean(ignoreInvalids); 1660 } 1661 1662 @Override 1663 public Dataset mean(int axis, boolean... ignoreInvalids) { 1664 return getStats().getMean(axis, ignoreInvalids); 1665 } 1666 1667 @Override 1668 public double variance() { 1669 return variance(false); 1670 } 1671 1672 @Override 1673 public double variance(boolean isWholePopulation, boolean... ignoreInvalids) { 1674 return getStats().getVariance(isWholePopulation, ignoreInvalids); 1675 } 1676 1677 @Override 1678 public Dataset variance(int axis) { 1679 return getStats().getVariance(axis, false); 1680 } 1681 1682 @Override 1683 public Dataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) { 1684 return getStats().getVariance(axis, isWholePopulation, ignoreInvalids); 1685 } 1686 1687 @Override 1688 public double stdDeviation() { 1689 return Math.sqrt(variance()); 1690 } 1691 1692 @Override 1693 public double stdDeviation(boolean isWholePopulation, boolean... ignoreInvalids) { 1694 return Math.sqrt(variance(isWholePopulation, ignoreInvalids)); 1695 } 1696 1697 @Override 1698 public Dataset stdDeviation(int axis) { 1699 return Maths.sqrt(variance(axis, false)); 1700 } 1701 1702 @Override 1703 public Dataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) { 1704 return Maths.sqrt(variance(axis, isWholePopulation, ignoreInvalids)); 1705 } 1706 1707 @Override 1708 public double rootMeanSquare(boolean... ignoreInvalids) { 1709 StatisticsMetadata<Number> stats = getStats(); 1710 final double mean = stats.getMean(ignoreInvalids).doubleValue(); 1711 final double var = stats.getVariance(true, ignoreInvalids); 1712 return Math.sqrt(var + mean * mean); 1713 } 1714 1715 @Override 1716 public Dataset rootMeanSquare(int axis, boolean... ignoreInvalids) { 1717 StatisticsMetadata<Number> stats = getStats(); 1718 Dataset v = stats.getVariance(axis, true, ignoreInvalids); 1719 Dataset m = stats.getMean(axis, ignoreInvalids); 1720 Dataset result = Maths.multiply(m, m); 1721 return Maths.sqrt(result.iadd(v)); 1722 } 1723 1724 /** 1725 * Set item from compatible dataset in a direct and speedy way. Remember to setDirty afterwards. 1726 * 1727 * @param dindex 1728 * @param sindex 1729 * @param src 1730 * is the source data buffer 1731 */ 1732 protected abstract void setItemDirect(final int dindex, final int sindex, final Object src); 1733 1734 /** 1735 * @return error broadcasted to current shape 1736 */ 1737 private Dataset getBroadcastedInternalError() { 1738 if (shape == null) { 1739 throw new IllegalArgumentException("Cannot get error for null dataset"); 1740 } 1741 ILazyDataset led = super.getErrors(); 1742 if (led == null) 1743 return null; 1744 1745 Dataset ed = null; 1746 try { 1747 ed = DatasetUtils.sliceAndConvertLazyDataset(led); 1748 } catch (DatasetException e) { 1749 logger.error("Could not get data from lazy dataset", e); 1750 } 1751 if (led != ed) { 1752 setErrors(ed); // set back 1753 } 1754 1755 return ed.getBroadcastView(shape); 1756 } 1757 1758 @Override 1759 public Dataset getErrors() { 1760 Dataset ed = getBroadcastedInternalError(); 1761 if (ed == null) 1762 return null; 1763 1764 return ed; 1765 } 1766 1767 @Override 1768 public double getError() { 1769 Dataset ed = getBroadcastedInternalError(); 1770 if (ed == null) 1771 return 0; 1772 1773 return ed.getDouble(); 1774 } 1775 1776 @Override 1777 public double getError(final int i) { 1778 Dataset ed = getBroadcastedInternalError(); 1779 if (ed == null) 1780 return 0; 1781 1782 return ed.getDouble(i); 1783 } 1784 1785 @Override 1786 public double getError(final int i, final int j) { 1787 Dataset ed = getBroadcastedInternalError(); 1788 if (ed == null) 1789 return 0; 1790 1791 return ed.getDouble(i, j); 1792 } 1793 1794 @Override 1795 public double getError(int... pos) { 1796 Dataset ed = getBroadcastedInternalError(); 1797 if (ed == null) 1798 return 0; 1799 1800 return ed.getDouble(pos); 1801 } 1802 1803 @Override 1804 public double[] getErrorArray(final int i) { 1805 Dataset ed = getBroadcastedInternalError(); 1806 if (ed == null) 1807 return null; 1808 1809 return new double[] {getError(i)}; 1810 } 1811 1812 @Override 1813 public double[] getErrorArray(final int i, final int j) { 1814 Dataset ed = getBroadcastedInternalError(); 1815 if (ed == null) 1816 return null; 1817 1818 return new double[] {getError(i, j)}; 1819 } 1820 1821 @Override 1822 public double[] getErrorArray(int... pos) { 1823 Dataset ed = getBroadcastedInternalError(); 1824 if (ed == null) 1825 return null; 1826 1827 return new double[] {getError(pos)}; 1828 } 1829 1830 protected Dataset getInternalSquaredError() { 1831 Dataset sed = getErrorBuffer().getBroadcastView(shape); 1832 return sed; 1833 } 1834 1835 @Override 1836 public Dataset getErrorBuffer() { 1837 ErrorMetadata emd = getErrorMetadata(); 1838 if (emd == null) 1839 return null; 1840 1841 if (!(emd instanceof ErrorMetadataImpl)) { 1842 ILazyDataset led = emd.getError(); 1843 Dataset ed; 1844 try { 1845 ed = DatasetUtils.sliceAndConvertLazyDataset(led); 1846 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1847 setMetadata(emd); 1848 emd.setError(ed); 1849 } catch (MetadataException me) { 1850 logger.error("Could not create metadata", me); 1851 } catch (DatasetException e) { 1852 logger.error("Could not get data from lazy dataset", e); 1853 } 1854 } 1855 1856 return ((ErrorMetadataImpl) emd).getSquaredError(); 1857 } 1858 1859 /** 1860 * Set a copy of the buffer that backs the (squared) error data 1861 * @param buffer can be null, anything that can be used to create a DoubleDataset or CompoundDoubleDataset 1862 */ 1863 @Override 1864 public void setErrorBuffer(Serializable buffer) { 1865 if (shape == null) { 1866 throw new IllegalArgumentException("Cannot set error buffer for null dataset"); 1867 } 1868 if (buffer == null) { 1869 clearMetadata(ErrorMetadata.class); 1870 return; 1871 } 1872 1873 IDataset d = (IDataset) createFromSerializable(buffer, false); 1874 ErrorMetadata emd = getErrorMetadata(); 1875 if (!(emd instanceof ErrorMetadataImpl)) { 1876 try { 1877 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1878 setMetadata(emd); 1879 } catch (MetadataException me) { 1880 logger.error("Could not create metadata", me); 1881 } 1882 } 1883 ((ErrorMetadataImpl) emd).setSquaredError(d); 1884 } 1885}