001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.io.Serializable;
016import java.lang.reflect.Array;
017import java.text.Format;
018import java.util.Arrays;
019import java.util.List;
020
021import org.eclipse.january.DatasetException;
022import org.eclipse.january.IMonitor;
023import org.eclipse.january.MetadataException;
024import org.eclipse.january.metadata.ErrorMetadata;
025import org.eclipse.january.metadata.MetadataFactory;
026import org.eclipse.january.metadata.StatisticsMetadata;
027import org.eclipse.january.metadata.internal.ErrorMetadataImpl;
028import org.eclipse.january.metadata.internal.StatisticsMetadataImpl;
029
030/**
031 * Generic container class for data 
032 * <p>
033 * Each subclass has an array of primitive types, elements of this array are grouped or
034 * compounded to make items 
035 * <p>
036 * Data items can be boolean, integer, float, complex float, vector float, etc
037 */
038public abstract class AbstractDataset extends LazyDatasetBase implements Dataset {
039        // pin UID to base class
040        private static final long serialVersionUID = Dataset.serialVersionUID;
041
042        protected int size; // number of items
043
044        transient protected AbstractDataset base; // is null when not a view
045        protected int[] stride; // can be null for row-major, contiguous datasets
046        protected int offset;
047
048        /**
049         * The data itself, held in a 1D array, but the object will wrap it to appear as possessing as many dimensions as
050         * wanted
051         */
052        protected Serializable odata = null;
053
054        /**
055         * Set aliased data as base data
056         */
057        abstract protected void setData();
058
059        /**
060         * Constructor required for serialisation.
061         */
062        public AbstractDataset() {
063        }
064
065        @Override
066        public synchronized Dataset synchronizedCopy() {
067                return clone();
068        }
069
070        @Override
071        public int hashCode() {
072                return getStats().getHash(shape);
073        }
074
075        @Override
076        abstract public AbstractDataset clone();
077
078        protected Format stringFormat = null;
079
080        @Override
081        public void setStringFormat(Format format) {
082                stringFormat = format;
083        }
084
085        @Override
086        public Dataset copy(final int dtype) {
087                if (getDType() == dtype) {
088                        return clone();
089                }
090                return DatasetUtils.copy(this, dtype);
091        }
092
093        @Override
094        public <T extends Dataset> T copy(Class<T> clazz) {
095                return DatasetUtils.copy(clazz, this);
096        }
097
098        @Override
099        public Dataset cast(final int dtype) {
100                if (getDType() == dtype) {
101                        return this;
102                }
103                return DatasetUtils.cast(this, dtype);
104        }
105
106        @Override
107        public <T extends Dataset> T cast(Class<T> clazz) {
108                return DatasetUtils.cast(clazz, this);
109        }
110
111        @Override
112        public Dataset cast(final boolean repeat, final int dtype, final int isize) {
113                if (getDType() == dtype && getElementsPerItem() == isize) {
114                        return this;
115                }
116                return DatasetUtils.cast(this, repeat, dtype, isize);
117        }
118
119        @Override
120        abstract public AbstractDataset getView(boolean deepCopyMetadata);
121
122        /**
123         * Copy fields from original to view
124         * @param orig
125         * @param view
126         * @param clone if true, then clone everything but bulk data
127         * @param cloneMetadata if true, clone metadata
128         */
129        protected static void copyToView(Dataset orig, AbstractDataset view, boolean clone, boolean cloneMetadata) {
130                view.name = orig.getName();
131                view.size = orig.getSize();
132                view.odata = orig.getBuffer();
133                view.offset = orig.getOffset();
134                view.base = orig instanceof AbstractDataset ? ((AbstractDataset) orig).base : null;
135
136                if (clone) {
137                        view.shape = orig.getShape();
138                        view.stride = orig instanceof AbstractDataset && ((AbstractDataset) orig).stride != null ?
139                                        ((AbstractDataset) orig).stride.clone() : null;
140                } else {
141                        view.shape = orig.getShapeRef();
142                        view.stride = orig instanceof AbstractDataset ? ((AbstractDataset) orig).stride : null;
143                }
144
145                view.metadata = getMetadataMap(orig, cloneMetadata);
146                int odtype = orig.getDType();
147                int vdtype = view.getDType();
148                if (odtype != vdtype) {
149                        view.setDirty();
150                }
151        }
152
153        @Override
154        public IntegerDataset getIndices() {
155                final IntegerDataset ret = DatasetUtils.indices(shape);
156                if (getName() != null) {
157                        ret.setName("Indices of " + getName());
158                }
159                return ret;
160        }
161
162        @Override
163        public Dataset getTransposedView(int... axes) {
164                axes = checkPermutatedAxes(shape, axes);
165
166                AbstractDataset t = getView(true);
167                if (axes == null || getRank() == 1)
168                        return t;
169
170                int rank = shape.length;
171                int[] tstride = new int[rank];
172                int[] toffset = new int[1];
173                int[] nshape = createStrides(new SliceND(shape), this, tstride, toffset);
174                int[] nstride = new int[rank];
175                for (int i = 0; i < rank; i++) {
176                        final int ax = axes[i];
177                        nstride[i] = tstride[ax];
178                        nshape[i] = shape[ax];
179                }
180                t.shape = nshape;
181                t.stride = nstride;
182                t.offset = toffset[0];
183                t.base = this;
184                t.setDirty();
185                t.transposeMetadata(axes);
186                return t;
187        }
188
189        @Override
190        public Dataset transpose(int... axes) {
191                Dataset t = getTransposedView(axes);
192                return t == null ? clone() : t.clone();
193        }
194
195        @Override
196        public Dataset swapAxes(int axis1, int axis2) {
197                int rank = shape.length;
198                axis1 = ShapeUtils.checkAxis(rank, axis1);
199                axis2 = ShapeUtils.checkAxis(rank, axis2);
200
201                if (rank == 1 || axis1 == axis2) {
202                        return this;
203                }
204
205                int[] axes = new int[rank];
206                for (int i = 0; i < rank; i++) {
207                        axes[i] = i;
208                }               
209
210                axes[axis1] = axis2;
211                axes[axis2] = axis1;
212                return getTransposedView(axes);
213        }
214
215        boolean isContiguous() {
216                if (stride == null)
217                        return true;
218
219                if (offset != 0)
220                        return false;
221
222                int s = getElementsPerItem();
223                for (int j = getRank() - 1; j >= 0; j--) {
224                        if (stride[j] != s) {
225                                return false;
226                        }
227                        s *= shape[j];
228                }
229
230                return true;
231        }
232
233        @Override
234        public Dataset flatten() {
235                if (!isContiguous()) { // need to make a copy if not contiguous
236                        return clone().flatten();
237                }
238                return reshape(size);
239        }
240
241        /**
242         * Fill dataset from object at depth dimension
243         * @param obj
244         * @param depth
245         * @param pos position
246         */
247        protected void fillData(Object obj, final int depth, final int[] pos) {
248                if (obj == null) {
249                        int dtype = getDType();
250                        if (dtype == FLOAT32)
251                                set(Float.NaN, pos);
252                        else if (dtype == FLOAT64)
253                                set(Double.NaN, pos);
254                        return;
255                }
256
257                Class<?> clazz = obj.getClass();
258                if (obj instanceof List<?>) {
259                        List<?> jl = (List<?>) obj;
260                        int l = jl.size();
261                        for (int i = 0; i < l; i++) {
262                                Object lo = jl.get(i);
263                                fillData(lo, depth + 1, pos);
264                                pos[depth]++;
265                        }
266                        pos[depth] = 0;
267                } else if (clazz.isArray()) {
268                        int l = Array.getLength(obj);
269                        if (clazz.equals(odata.getClass())) {
270                                System.arraycopy(obj, 0, odata, get1DIndex(pos), l);
271                        } else if (clazz.getComponentType().isPrimitive()) {
272                                for (int i = 0; i < l; i++) {
273                                        set(Array.get(obj, i), pos);
274                                        pos[depth]++;
275                                }
276                                pos[depth] = 0;
277                        } else {
278                                for (int i = 0; i < l; i++) {
279                                        fillData(Array.get(obj, i), depth + 1, pos);
280                                        pos[depth]++;
281                                }
282                                pos[depth] = 0;
283                        }
284                } else if (obj instanceof IDataset) {
285                        boolean[] a = new boolean[shape.length];
286                        for (int i = depth; i < a.length; i++)
287                                a[i] = true;
288                        setSlice(obj, getSliceIteratorFromAxes(pos, a));
289                } else {
290                        set(obj, pos);
291                }
292        }
293
294        @Override
295        public IndexIterator getIterator(final boolean withPosition) {
296                if (stride != null) {
297                        return base.getSize() == 1  ? (withPosition ? new PositionIterator(offset, shape) :
298                                new SingleItemIterator(offset, size)) : new StrideIterator(shape, stride, offset);
299                }
300                if (shape == null) {
301                        return new NullIterator(shape, shape);
302                }
303                
304                return withPosition ? new ContiguousIteratorWithPosition(shape, size) : new ContiguousIterator(size);
305        }
306
307        @Override
308        public IndexIterator getIterator() {
309                return getIterator(false);
310        }
311
312        @Override
313        public PositionIterator getPositionIterator(final int... axes) {
314                return new PositionIterator(shape, axes);
315        }
316
317        @Override
318        public IndexIterator getSliceIterator(final int[] start, final int[] stop, final int[] step) {
319                return getSliceIterator(new SliceND(shape, start, stop, step));
320        }
321
322        /**
323         * @param slice
324         * @return an slice iterator that operates like an IndexIterator
325         */
326        public IndexIterator getSliceIterator(SliceND slice) {
327                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
328                        return new NullIterator(shape, slice.getShape());
329                }
330                if (stride != null) {
331                        return new StrideIterator(getElementsPerItem(), shape, stride, offset, slice);
332                }
333                return new SliceIterator(shape, size, slice);
334        }
335
336        @Override
337        public SliceIterator getSliceIteratorFromAxes(final int[] pos, boolean[] axes) {
338                int rank = shape.length;
339                int[] start;
340                int[] stop = new int[rank];
341                int[] step = new int[rank];
342
343                if (pos == null) {
344                        start = new int[rank];
345                } else if (pos.length == rank) {
346                        start = pos.clone();
347                } else {
348                        throw new IllegalArgumentException("pos array length is not equal to rank of dataset");
349                }
350                if (axes == null) {
351                        axes = new boolean[rank];
352                        Arrays.fill(axes, true);
353                } else if (axes.length != rank) {
354                        throw new IllegalArgumentException("axes array length is not equal to rank of dataset");
355                }
356
357                for (int i = 0; i < rank; i++) {
358                        if (axes[i]) {
359                                stop[i] = shape[i];
360                        } else {
361                                stop[i] = start[i] + 1;
362                        }
363                        step[i] = 1;
364                }
365                return (SliceIterator) getSliceIterator(start, stop, step);
366        }
367
368        @Override
369        public BooleanIterator getBooleanIterator(Dataset choice) {
370                return getBooleanIterator(choice, true);
371        }
372
373        @Override
374        public BooleanIterator getBooleanIterator(Dataset choice, boolean value) {
375                return BooleanIterator.createIterator(value, this, choice, this);
376        }
377
378        @Override
379        public Dataset getByBoolean(Dataset selection) {
380                checkCompatibility(selection);
381
382                final int length = ((Number) selection.sum()).intValue();
383                final int is = getElementsPerItem();
384                Dataset r = DatasetFactory.zeros(is, getClass(), length);
385                BooleanIterator biter = getBooleanIterator(selection);
386
387                int i = 0;
388                while (biter.hasNext()) {
389                        r.setObjectAbs(i, getObjectAbs(biter.index));
390                        i += is;
391                }
392                return r;
393        }
394
395        @Override
396        public Dataset getBy1DIndex(IntegerDataset index) {
397                final int is = getElementsPerItem();
398                final Dataset r = DatasetFactory.zeros(is, getClass(), index.getShape());
399                final IntegerIterator iter = new IntegerIterator(index, size, is);
400
401                int i = 0;
402                while (iter.hasNext()) {
403                        r.setObjectAbs(i, getObjectAbs(iter.index));
404                        i += is;
405                }
406                return r;
407        }
408
409        @Override
410        public Dataset getByIndexes(final Object... indexes) {
411                final IntegersIterator iter = new IntegersIterator(shape, indexes);
412                final int is = getElementsPerItem();
413                final Dataset r = DatasetFactory.zeros(is, getClass(), iter.getShape());
414
415                final int[] pos = iter.getPos();
416                int i = 0;
417                while (iter.hasNext()) {
418                        r.setObjectAbs(i, getObject(pos));
419                        i += is;
420                }
421                return r;
422        }
423
424        @Override
425        public Class<?> getElementClass() {
426                return DTypeUtils.getElementClass(getDType());
427        }
428
429        @Override
430        public boolean hasFloatingPointElements() {
431                Class<?> cls = getElementClass();
432                return cls == Float.class || cls == Double.class;
433        }
434
435        @Override
436        public int getElementsPerItem() {
437                return DTypeUtils.getElementsPerItem(getDType());
438        }
439
440        @Override
441        public int getItemBytes() {
442                return DTypeUtils.getItemBytes(getDType(), getElementsPerItem());
443        }
444
445        @Override
446        public int getSize() {
447                return size;
448        }
449
450        @Override
451        public int[] getShape() {
452                // make a copy of the dimensions data, and put that out
453                if (shape == null) {
454                        logger.warn("Shape is null!!!");
455                        return new int[] {};
456                }
457                return shape.clone();
458        }
459
460        @Override
461        public int getRank() {
462                return shape == null ? 0 : shape.length;
463        }
464
465        @Override
466        public int getNbytes() {
467                return getSize() * getItemBytes();
468        }
469
470        /**
471         * Check for -1 placeholder in shape and replace if necessary
472         * @param shape
473         * @param size
474         */
475        private void checkShape(int[] shape, int size) {
476                int rank = shape.length;
477                int found = -1;
478                int nsize = 1;
479                for (int i = 0; i < rank; i++) {
480                        int d = shape[i];
481                        if (d == -1) {
482                                if (found == -1) {
483                                        found = i;
484                                } else {
485                                        logger.error("Can only have one -1 placeholder in shape");
486                                        throw new IllegalArgumentException("Can only have one -1 placeholder in shape");
487                                }
488                        } else {
489                                nsize *= d;
490                        }
491                }
492                if (found >= 0) {
493                        shape[found] = size/nsize;
494                } else if (nsize != size && !(rank == 0 && size == 0)) {
495                        logger.error("New shape is not same size as old shape");
496                        throw new IllegalArgumentException("New size is not same as the old size. Old size is "+size+" new size is "+nsize+" and shape is "+Arrays.toString(shape));
497                }
498        }
499
500        @Override
501        public void setShape(final int... shape) {
502                int[] nshape = shape.clone();
503                checkShape(nshape, size);
504                if (Arrays.equals(this.shape, nshape)) {
505                        return;
506                }
507
508                if (stride != null) {
509                        // the only compatible shapes are ones where new dimensions are factors of old dimensions
510                        // or are combined adjacent old dimensions 
511                        int[] oshape = this.shape;
512                        int orank = oshape.length;
513                        int nrank = nshape.length;
514                        int diff = nrank - orank;
515                        int[] nstride = new int[nrank];
516                        boolean ones = true;
517                        // work forwards for broadcasting cases
518                        for (int i = 0, j = 0; i < orank || j < nrank;) {
519                                if (j >= diff && i < orank && j < nrank && oshape[i] == nshape[j]) {
520                                        nstride[j++] = stride[i++];
521                                } else if (j < nrank && nshape[j] == 1) {
522                                        nstride[j++] = 0;
523                                } else if (i < orank && oshape[i] == 1) {
524                                        i++;
525                                } else {
526                                        if (j < nrank)
527                                                j++;
528                                        if (i < orank)
529                                                i++;
530                                        ones = false;
531                                }
532                        }
533                        if (!ones) { // not just ones differ in shapes
534                                int[] ostride = stride;
535                                int ob = 0;
536                                int oe = 1;
537                                int nb = 0;
538                                int ne = 1;
539                                while (ob < orank && nb < nrank) {
540                                        int ol = oshape[ob];
541                                        int nl = nshape[nb];
542                                        
543                                        if (nl < ol) { // find group of shape dimensions that form common size
544                                                do { // case where new shape spreads single dimension over several dimensions
545                                                        if (ne == nrank) {
546                                                                break;
547                                                        }
548                                                        nl *= nshape[ne++];
549                                                } while (nl < ol);
550                                                if (nl != ol) {
551                                                        logger.error("Subshape is incompatible with single dimension");
552                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
553                                                }
554                                                int on = ne - 1;
555                                                while (nshape[on] == 1) {
556                                                        on--;
557                                                }
558
559                                                nstride[on] = ostride[ob];
560                                                for (int n = on - 1; n >= nb; n--) {
561                                                        if (nshape[n] == 1)
562                                                                continue;
563
564                                                        nstride[n] = nshape[on] * nstride[on];
565                                                        on = n;
566                                                }
567                                        } else if (ol < nl) {
568                                                do { // case where new shape combines several dimensions into one dimension
569                                                        if (oe == orank) {
570                                                                break;
571                                                        }
572                                                        ol *= oshape[oe++];
573                                                } while (ol < nl);
574                                                if (nl != ol) {
575                                                        logger.error("Single dimension is incompatible with subshape");
576                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
577                                                }
578
579                                                int oo = oe - 1;
580                                                while (oshape[oo] == 1) {
581                                                        oo--;
582                                                }
583                                                int os = ostride[oo];
584                                                for (int o = oo - 1; o >= ob; o--) {
585                                                        if (oshape[o] == 1)
586                                                                continue;
587                                                        if (ostride[o] != oshape[oo] * ostride[oo]) {
588                                                                logger.error("Subshape cannot be a non-contiguous view");
589                                                                throw new IllegalArgumentException("Subshape cannot be a non-contiguous view");
590                                                        }
591                                                        oo = o;
592                                                }
593                                                nstride[nb] = os;
594                                        } else {
595                                                nstride[nb] = ostride[ob];
596                                        }
597
598                                        ob = oe++;
599                                        nb = ne++;
600                                }
601                        }
602        
603                        stride = nstride;
604                }
605
606                setDirty();
607                if (this.shape != null && metadata != null) {
608                        reshapeMetadata(this.shape, nshape);
609                }
610                this.shape = nshape;
611        }
612
613        @Override
614        public int[] getShapeRef() {
615                return shape;
616        }
617
618        @Override
619        public int getOffset() {
620                return offset;
621        }
622
623        @Override
624        public int[] getStrides() {
625                return stride;
626        }
627
628        @Override
629        public Serializable getBuffer() {
630                return odata;
631        }
632
633        @Override
634        public void overrideInternal(Serializable buffer, int... shape) {
635                if (buffer != null) {
636                        odata = buffer;
637                        setData();
638                        setDirty();
639                }
640        
641                if (shape != null) {
642                        this.shape = shape.clone();
643                        size = ShapeUtils.calcSize(this.shape);
644                }
645        }
646
647        /**
648         * Create a stride array from dataset
649         * @param a dataset
650         * @param offset output offset
651         * @return new strides
652         */
653        public static int[] createStrides(Dataset a, final int[] offset) {
654                return createStrides(a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), offset);
655        }
656
657        /**
658         * Create a stride array from dataset
659         * @param isize
660         * @param shape
661         * @param oStride original stride
662         * @param oOffset original offset (only used if there is an original stride)
663         * @param offset output offset
664         * @return new strides
665         */
666        public static int[] createStrides(final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] offset) {
667                int rank = shape.length;
668                final int[] stride;
669                if (oStride == null) {
670                        offset[0] = 0;
671                        stride = new int[rank];
672                        int s = isize;
673                        for (int j = rank - 1; j >= 0; j--) {
674                                stride[j] = s;
675                                s *= shape[j];
676                        }
677                } else {
678                        offset[0] = oOffset;
679                        stride = oStride.clone();
680                }
681                return stride;
682        }
683
684        /**
685         * Create a stride array from slice information and a dataset
686         * @param slice
687         * @param a dataset
688         * @param stride output stride
689         * @param offset output offset
690         * @return new shape
691         */
692        public static int[] createStrides(final SliceND slice, final Dataset a, final int[] stride, final int[] offset) {
693                return createStrides(slice, a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), stride, offset);
694        }
695
696        /**
697         * Create a stride array from slice and dataset information
698         * @param slice
699         * @param isize
700         * @param shape
701         * @param oStride original stride
702         * @param oOffset original offset (only used if there is an original stride)
703         * @param stride output stride
704         * @param offset output offset
705         * @return new shape
706         */
707        public static int[] createStrides(final SliceND slice, final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] stride, final int[] offset) {
708                int[] lstart = slice.getStart();
709                int[] lstep = slice.getStep();
710                int[] newShape = slice.getShape();
711                int rank = shape.length;
712
713                if (oStride == null) {
714                        int s = isize;
715                        offset[0] = 0;
716                        for (int j = rank - 1; j >= 0; j--) {
717                                stride[j] = s * lstep[j];
718                                offset[0] += s * lstart[j];
719                                s *= shape[j];
720                        }
721                } else {
722                        offset[0] = oOffset;
723                        for (int j = 0; j < rank; j++) {
724                                int s = oStride[j];
725                                stride[j] = lstep[j] * s;
726                                offset[0] += lstart[j] * s;
727                        }
728                }
729
730                return newShape;
731        }
732
733        @Override
734        public Dataset getBroadcastView(int... broadcastShape) {
735                AbstractDataset view = getView(true);
736                
737                if (!Arrays.equals(shape, broadcastShape)) {
738                        List<int[]> nShapes = BroadcastUtils.broadcastShapesToMax(broadcastShape, shape);
739                        view.setShape(nShapes.get(0));
740                        view.stride = BroadcastUtils.createBroadcastStrides(view, broadcastShape);
741                        view.base = this;
742                        view.shape = broadcastShape.clone();
743                        view.size = ShapeUtils.calcSize(broadcastShape);
744                        if (view.name == null || view.name.isEmpty()) {
745                                view.name = "Broadcast from " + Arrays.toString(shape);
746                        } else {
747                                view.name = "Broadcast of " + view.name + " from " + Arrays.toString(shape);
748                        }
749                }
750                return view;
751        }
752
753        @Override
754        public Dataset getSliceView(final int[] start, final int[] stop, final int[] step) {
755                return getSliceView(new SliceND(shape, start, stop, step));
756        }
757
758        @Override
759        public Dataset getSliceView(Slice... slice) {
760                if (slice == null || slice.length == 0) {
761                        return getView(true);
762                }
763
764                return getSliceView(new SliceND(shape, slice));
765        }
766
767        /**
768         * Get a slice of the dataset. The returned dataset is a view on a selection of items
769         * @param slice
770         * @return slice view
771         */
772        @Override
773        public Dataset getSliceView(SliceND slice) {
774                if (slice.isAll()) {
775                        return getView(true);
776                }
777
778                final int rank = shape.length;
779                int[] sStride = new int[rank];
780                int[] sOffset = new int[1];
781
782                int[] sShape = createStrides(slice, this, sStride, sOffset);
783        
784                AbstractDataset s = getView(false);
785                s.shape = sShape;
786                s.size = ShapeUtils.calcSize(sShape);
787                s.stride = sStride;
788                s.offset = sOffset[0];
789                s.base = this;
790
791                s.metadata = copyMetadata();
792                s.sliceMetadata(true, slice);
793
794                s.setDirty();
795                s.setName(name + BLOCK_OPEN + slice + BLOCK_CLOSE);
796
797                return s;
798        }
799
800        /**
801         * Get flattened view index of given position 
802         * @param pos
803         *            the integer array specifying the n-D position
804         * @return the index on the flattened dataset
805         */
806        private int getFlat1DIndex(final int[] pos) {
807                final int imax = pos.length;
808                if (imax == 0) {
809                        return 0;
810                }
811
812                return get1DIndexFromShape(pos);
813        }
814
815        /**
816         * @since 2.0
817         */
818        protected int getFirst1DIndex() {
819                if (shape == null) {
820                        throw new IllegalArgumentException("Cannot find an index from a null shape");
821                }
822                return stride == null ? 0 : offset;
823        }
824
825        @Override
826        public int get1DIndex(final int... n) {
827                if (n.length == 0 && shape.length == 0)
828                        return offset;
829
830                return stride == null ? get1DIndexFromShape(n) : get1DIndexFromStrides(n);
831        }
832
833        private static void throwAIOOBException(int i, int s, int d) {
834                throw new ArrayIndexOutOfBoundsException("Index (" + i + ") out of range [-" + s + "," + s
835                                + "] in dimension " + d);
836        }
837
838        /**
839         * @param i
840         * @return the index on the data array corresponding to that location
841         */
842        protected int get1DIndex(int i) {
843                if (shape == null) {
844                        throw new IllegalArgumentException("Cannot find an index from a null shape");
845                }
846                if (shape.length > 1) {
847                        logger.error("This dataset is not 1D but was addressed as such");
848                        throw new UnsupportedOperationException("This dataset is not 1D but was addressed as such");
849                }
850                if (i < 0) {
851                        i += shape[0];
852                }
853                if (i < 0 || i >= shape[0]) {
854                        throwAIOOBException(i, shape[0], 0);
855                }
856                return stride == null ? i : i*stride[0] + offset;
857        }
858
859        /**
860         * @param i
861         * @param j
862         * @return the index on the data array corresponding to that location
863         */
864        protected int get1DIndex(int i, int j) {
865                if (shape == null) {
866                        throw new IllegalArgumentException("Cannot find an index from a null shape");
867                }
868                if (shape.length != 2) {
869                        logger.error("This dataset is not 2D but was addressed as such");
870                        throw new UnsupportedOperationException("This dataset is not 2D but was addressed as such");
871                }
872                if (i < 0) {
873                        i += shape[0];
874                }
875                if (i < 0 || i >= shape[0]) {
876                        throwAIOOBException(i, shape[0], 0);
877                }
878                if (j < 0) {
879                        j += shape[1];
880                }
881                if (j < 0 || j >= shape[1]) {
882                        throwAIOOBException(i, shape[1], 1);
883                }
884                return stride == null ? i*shape[1] + j : i*stride[0] + j*stride[1] + offset;
885        }
886
887        protected int get1DIndexFromShape(final int[] n) {
888                return get1DIndexFromShape(shape, n);
889        }
890
891        protected static int get1DIndexFromShape(final int[] shape, final int[] n) {
892                if (shape == null) {
893                        throw new IllegalArgumentException("Cannot find an index from a null shape");
894                }
895                final int rank = shape.length;
896                if (rank != n.length) {
897                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
898                        logger.error(errMsg);
899                        throw new IllegalArgumentException(errMsg);
900                }
901                int index = 0;
902                for (int i = 0; i < rank; i++) {
903                        final int si = shape[i];
904                        int ni = n[i];
905                        if (ni < 0) {
906                                ni += si;
907                        }
908                        if (ni < 0 || ni >= si) {
909                                throwAIOOBException(ni, si, i);
910                        }
911                        index = index * si + ni;
912                }
913
914                return index;
915        }
916
917        private int get1DIndexFromStrides(final int[] n) {
918                return get1DIndexFromStrides(shape, stride, offset, n);
919        }
920
921        private static int get1DIndexFromStrides(final int[] shape, final int[] stride, final int offset, final int[] n) {
922                if (shape == null) {
923                        throw new IllegalArgumentException("Cannot find an index from a null shape");
924                }
925                final int rank = shape.length;
926                if (rank != n.length) {
927                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
928                        logger.error(errMsg);
929                        throw new IllegalArgumentException(errMsg);
930                }
931                int index = offset;
932                for (int i = 0; i < rank; i++) {
933                        final int st = stride[i];
934                        if (st != 0) { // not broadcasted
935                                final int si = shape[i];
936                                int ni = n[i];
937                                if (ni < 0) {
938                                        ni += si;
939                                }
940                                if (ni < 0 || ni >= si) {
941                                        throwAIOOBException(ni, si, i);
942                                }
943                                index += st * ni;
944                        }
945                }
946                return index;
947        }
948
949        @Override
950        public int[] getNDPosition(final int n) {
951                if (isIndexInRange(n)) {
952                        throw new IllegalArgumentException("Index provided " + n
953                                        + "is larger then the size of the containing array");
954                }
955
956                return stride == null ? ShapeUtils.getNDPositionFromShape(n, shape) : getNDPositionFromStrides(n);
957        }
958
959        private boolean isIndexInRange(final int n) {
960                if (stride == null) {
961                        return n >= size;
962                }
963                return n >= getBufferLength();
964        }
965
966        /**
967         * @return entire buffer length
968         */
969        abstract protected int getBufferLength();
970
971        private int[] getNDPositionFromStrides(int n) {
972                n -= offset;
973                int rank = shape.length;
974                if (rank == 1) {
975                        return new int[] { n / stride[0] };
976                }
977
978                int[] output = new int[rank];
979                int i = 0;
980                while (i != n) { // TODO find more efficient way than this exhaustive search
981                        int j = rank - 1;
982                        for (; j >= 0; j--) {
983                                output[j]++;
984                                i += stride[j];
985                                if (output[j] >= shape[j]) {
986                                        output[j] = 0;
987                                        i -= shape[j] * stride[j];
988                                } else {
989                                        break;
990                                }
991                        }
992                        if (j == -1) {
993                                logger.error("Index was not found in this strided dataset");
994                                throw new IllegalArgumentException("Index was not found in this strided dataset");
995                        }
996                }
997
998                return output;
999        }
1000
1001        @Override
1002        public int checkAxis(int axis) {
1003                return ShapeUtils.checkAxis(shape.length, axis);
1004        }
1005
1006        @Deprecated
1007        protected static int checkAxis(int rank, int axis) {
1008                return ShapeUtils.checkAxis(rank, axis);
1009        }
1010
1011        protected static final char BLOCK_OPEN = '[';
1012        protected static final char BLOCK_CLOSE = ']';
1013
1014        @Override
1015        public String toString() {
1016                final int rank = shape == null ? 0 : shape.length;
1017                final StringBuilder out = new StringBuilder();
1018
1019                if (DTypeUtils.isDTypeElemental(getDType())) {
1020                        out.append("Dataset ");
1021                } else {
1022                        out.append("Compound dataset (");
1023                        out.append(getElementsPerItem());
1024                        out.append(") ");
1025                }
1026
1027                if (name != null && name.length() > 0) {
1028                        out.append("'");
1029                        out.append(name);
1030                        out.append("' has shape ");
1031                } else {
1032                        out.append("shape is ");
1033                }
1034
1035                out.append(BLOCK_OPEN);
1036                if (rank > 0) {
1037                        out.append(shape[0]);
1038                }
1039                for (int i = 1; i < rank; i++) {
1040                        out.append(", " + shape[i]);
1041                }
1042                out.append(BLOCK_CLOSE);
1043                return out.toString();
1044        }
1045
1046        @Override
1047        public String toString(boolean showData) {
1048                if (!showData) {
1049                        return toString();
1050                }
1051
1052                if (size == 0) {
1053                        return "[]";
1054                }
1055
1056                final int rank = shape == null ? 0 : shape.length;
1057                final StringBuilder out = new StringBuilder();
1058
1059                if (rank > 0) {
1060                        int[] pos = new int[rank];
1061                        final StringBuilder lead = new StringBuilder();
1062                        printBlocks(out, lead, 0, pos);
1063                } else {
1064                        out.append(getString());
1065                }
1066                return out.toString();
1067        }
1068
1069        /**
1070         * Limit to strings output via the toString() method
1071         */
1072        private static int maxStringLength = 120;
1073
1074        /**
1075         * Set maximum line length for toString() method
1076         * @param maxLineLength
1077         */
1078        public static void setMaxLineLength(int maxLineLength) {
1079                maxStringLength = maxLineLength;
1080        }
1081
1082        /**
1083         * @return maximum line length for toString() method
1084         */
1085        public static int getMaxLineLength() {
1086                return maxStringLength;
1087        }
1088
1089        /**
1090         * Limit to number of sub-blocks output via the toString() method
1091         */
1092        private static final int MAX_SUBBLOCKS = 6;
1093
1094        private final static String SEPARATOR = ",";
1095        private final static String SPACE = " ";
1096        private final static String ELLIPSIS = "...";
1097        private final static String NEWLINE = "\n";
1098
1099        /**
1100         * Make a line of output for last dimension of dataset
1101         * 
1102         * @param start
1103         * @return line
1104         */
1105        private StringBuilder makeLine(final int end, final int[] start) {
1106                StringBuilder line = new StringBuilder();
1107                final int[] pos;
1108                if (end >= start.length) {
1109                        pos = Arrays.copyOf(start, end + 1);
1110                } else {
1111                        pos = start;
1112                }
1113                pos[end] = 0;
1114                line.append(BLOCK_OPEN);
1115                line.append(getString(pos));
1116
1117                final int length = shape[end];
1118
1119                // trim elements printed if length exceed estimate of maximum elements
1120                int excess = length - maxStringLength / 3; // space + number + separator
1121                int midIndex = -1;
1122                if (excess > 0) {
1123                        int index = (length - excess) / 2;
1124                        for (int y = 1; y < index; y++) {
1125                                line.append(SEPARATOR + SPACE);
1126                                pos[end] = y;
1127                                line.append(getString(pos));
1128                        }
1129                        midIndex = line.length() + 2;
1130                        index = (length + excess) / 2;
1131                        for (int y = index; y < length; y++) {
1132                                line.append(SEPARATOR + SPACE);
1133                                pos[end] = y;
1134                                line.append(getString(pos));
1135                        }
1136                } else {
1137                        for (int y = 1; y < length; y++) {
1138                                line.append(SEPARATOR + SPACE);
1139                                pos[end] = y;
1140                                line.append(getString(pos));
1141                        }
1142                }
1143                line.append(BLOCK_CLOSE);
1144
1145                // trim string down to limit
1146                int lineLength = line.length();
1147                excess = lineLength - maxStringLength - ELLIPSIS.length() - 1;
1148                if (excess > 0) {
1149                        int index = (lineLength - excess) / 2;
1150                        if (midIndex > 0 && index > midIndex) {
1151                                index = midIndex;
1152                        } else {
1153                                index = line.lastIndexOf(SEPARATOR, index) + 2;
1154                        }
1155                        StringBuilder out = new StringBuilder(line.subSequence(0, index));
1156                        out.append(ELLIPSIS + SEPARATOR);
1157                        index = (lineLength + excess) / 2;
1158                        if (midIndex > 0 && index <= midIndex) {
1159                                index = midIndex - 1;
1160                        } else {
1161                                index = line.indexOf(SEPARATOR, index) + 1;
1162                        }
1163                        out.append(line.subSequence(index, lineLength));
1164                        return out;
1165                } else if (midIndex > 0) { // add ellipsis
1166                        StringBuilder out = new StringBuilder(line.subSequence(0, midIndex));
1167                        out.append(ELLIPSIS + SEPARATOR + SPACE);
1168                        out.append(line.subSequence(midIndex, lineLength));
1169                        return out;
1170                }
1171
1172                return line;
1173        }
1174
1175        /**
1176         * recursive method to print blocks
1177         */
1178        private void printBlocks(final StringBuilder out, final StringBuilder lead, final int level, final int[] pos) {
1179                if (out.length() > 0) {
1180                        char last = out.charAt(out.length() - 1);
1181                        if (last != BLOCK_OPEN) {
1182                                out.append(lead);
1183                        }
1184                }
1185                final int end = getRank() - 1;
1186                if (level != end) {
1187                        out.append(BLOCK_OPEN);
1188                        int length = shape[level];
1189
1190                        // first sub-block
1191                        pos[level] = 0;
1192                        StringBuilder newlead = new StringBuilder(lead);
1193                        newlead.append(SPACE);
1194                        printBlocks(out, newlead, level + 1, pos);
1195                        if (length < 2) { // escape
1196                                out.append(BLOCK_CLOSE);
1197                                return;
1198                        }
1199
1200                        out.append(SEPARATOR + NEWLINE);
1201                        for (int i = level + 1; i < end; i++) {
1202                                out.append(NEWLINE);
1203                        }
1204
1205                        // middle sub-blocks
1206                        if (length < MAX_SUBBLOCKS) {
1207                                for (int x = 1; x < length - 1; x++) {
1208                                        pos[level] = x;
1209                                        printBlocks(out, newlead, level + 1, pos);
1210                                        if (end <= level + 1) {
1211                                                out.append(SEPARATOR + NEWLINE);
1212                                        } else {
1213                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1214                                        }
1215                                }
1216                        } else {
1217                                final int excess = length - MAX_SUBBLOCKS;
1218                                int xmax = (length - excess) / 2;
1219                                for (int x = 1; x < xmax; x++) {
1220                                        pos[level] = x;
1221                                        printBlocks(out, newlead, level + 1, pos);
1222                                        if (end <= level + 1) {
1223                                                out.append(SEPARATOR + NEWLINE);
1224                                        } else {
1225                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1226                                        }
1227                                }
1228                                out.append(newlead);
1229                                out.append(ELLIPSIS + SEPARATOR + NEWLINE);
1230                                xmax = (length + excess) / 2;
1231                                for (int x = xmax; x < length - 1; x++) {
1232                                        pos[level] = x;
1233                                        printBlocks(out, newlead, level + 1, pos);
1234                                        if (end <= level + 1) {
1235                                                out.append(SEPARATOR + NEWLINE);
1236                                        } else {
1237                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1238                                        }
1239                                }
1240                        }
1241
1242                        // last sub-block
1243                        pos[level] = length - 1;
1244                        printBlocks(out, newlead, level + 1, pos);
1245                        out.append(BLOCK_CLOSE);
1246                } else {
1247                        out.append(makeLine(end, pos));
1248                }
1249        }
1250
1251        @Override
1252        public Dataset squeezeEnds() {
1253                return squeeze(true);
1254        }
1255
1256        @Override
1257        public Dataset squeeze() {
1258                return squeeze(false);
1259        }
1260
1261        @Override
1262        public Dataset squeeze(boolean onlyFromEnds) {
1263                final int[] tshape = ShapeUtils.squeezeShape(shape, onlyFromEnds);
1264                final int[] oshape = shape;
1265                if (stride == null) {
1266                        shape = tshape;
1267                } else {
1268                        int rank = shape.length;
1269                        int trank = tshape.length;
1270                        if (trank < rank) {
1271                                int[] tstride = new int[tshape.length];
1272                                if (onlyFromEnds) {
1273                                        for (int i = 0; i < rank; i++) {
1274                                                if (shape[i] != 1) {
1275                                                        for (int k = 0; k < trank; k++) {
1276                                                                tstride[k] = stride[i++];
1277                                                        }
1278                                                        break;
1279                                                }
1280                                        }
1281                                } else {
1282                                        int t = 0;
1283                                        for (int i = 0; i < rank; i++) {
1284                                                if (shape[i] != 1) {
1285                                                        tstride[t++] = stride[i];
1286                                                }
1287                                        }
1288                                }
1289                                shape = tshape;
1290                                stride = tstride;
1291                        }
1292                }
1293
1294                setDirty();
1295                reshapeMetadata(oshape, shape);
1296                return this;
1297        }
1298
1299        @Override
1300        public boolean isCompatibleWith(final ILazyDataset g) {
1301                return ShapeUtils.areShapesCompatible(shape, g.getShape());
1302        }
1303
1304        @Override
1305        public void checkCompatibility(final ILazyDataset g) throws IllegalArgumentException {
1306                ShapeUtils.checkCompatibility(this, g);
1307        }
1308
1309        @Override
1310        public Dataset reshape(final int... shape) {
1311                Dataset a;
1312                try {
1313                        a = getView(true);
1314                        a.setShape(shape);
1315                } catch (IllegalArgumentException e) {
1316                        a = clone();
1317                        a.setShape(shape);
1318                }
1319                return a;
1320        }
1321
1322        /**
1323         * @param start
1324         * @param stop
1325         * @param step
1326         * @return number of steps to take
1327         */
1328        protected static int calcSteps(final double start, final double stop, final double step) {
1329                return Math.max(0, (int) Math.ceil((stop - start) / step));
1330        }
1331
1332        @Override
1333        public boolean isComplex() {
1334                int type = getDType();
1335                return type == COMPLEX64 || type == COMPLEX128;
1336        }
1337
1338        @Override
1339        public Dataset getRealPart() {
1340                return this;
1341        }
1342
1343        @Override
1344        public Dataset getRealView() {
1345                return getView(true);
1346        }
1347
1348        @Override
1349        public Dataset getSlice(final int[] start, final int[] stop, final int[] step) {
1350                return getSlice(new SliceND(shape, start, stop, step));
1351        }
1352
1353        @Override
1354        public Dataset getSlice(Slice... slice) {
1355                return getSlice(new SliceND(shape, slice));
1356        }
1357
1358        @Override
1359        public Dataset getSlice(IMonitor monitor, Slice... slice) {
1360                return getSlice(slice);
1361        }
1362
1363        @Override
1364        public Dataset getSlice(IMonitor monitor, SliceND slice) {
1365                return getSlice(slice);
1366        }
1367
1368        @Override
1369        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) {
1370                return getSlice(start, stop, step);
1371        }
1372
1373        /**
1374         * Get a slice of the dataset. The returned dataset is a copied selection of items
1375         * @param slice
1376         * @return The dataset of the sliced data
1377         */
1378        @Override
1379        public Dataset getSlice(final SliceND slice) {
1380                SliceIterator it = (SliceIterator) getSliceIterator(slice);
1381                AbstractDataset s = getSlice(it);
1382                s.metadata = copyMetadata();
1383                s.setDirty();
1384                s.sliceMetadata(true, slice);
1385                return s;
1386        }
1387
1388        /**
1389         * Get a slice of the dataset. The returned dataset is a copied selection of items
1390         * 
1391         * @param iterator Slice iterator
1392         * @return The dataset of the sliced data
1393         */
1394        abstract public AbstractDataset getSlice(final SliceIterator iterator);
1395
1396        @SuppressWarnings("deprecation")
1397        @Override
1398        public Dataset setSlice(final Object obj, final SliceND slice) {
1399                Dataset ds;
1400                if (obj instanceof Dataset) {
1401                        ds = (Dataset) obj;
1402                } else if (obj instanceof IDataset) {
1403                        ds = DatasetUtils.convertToDataset((IDataset) obj);
1404                } else {
1405                        int dtype = getDType();
1406                        if (dtype != Dataset.BOOL) {
1407                                dtype = DTypeUtils.getLargestDType(dtype);
1408                        }
1409                        ds = DatasetFactory.createFromObject(getElementsPerItem(), dtype, obj);
1410                }
1411
1412                return setSlicedView(getSliceView(slice), ds);
1413        }
1414
1415        @Override
1416        public Dataset setSlice(final Object obj, final int[] start, final int[] stop, final int[] step) {
1417                return setSlice(obj, new SliceND(shape, start, stop, step));
1418        }
1419
1420        /**
1421         * Set a view of current dataset to given dataset with broadcasting
1422         * @param view
1423         * @param d
1424         * @return this dataset
1425         */
1426        abstract Dataset setSlicedView(Dataset view, Dataset d);
1427
1428        @Override
1429        public Dataset setSlice(Object obj, Slice... slice) {
1430                if (slice == null || slice.length == 0) {
1431                        return setSlice(obj, new SliceND(shape));
1432                }
1433                return setSlice(obj, new SliceND(shape, slice));
1434        }
1435
1436        @Override
1437        public boolean all() {
1438                return Comparisons.allTrue(this);
1439        }
1440
1441        @Override
1442        public BooleanDataset all(final int axis) {
1443                return Comparisons.allTrue(this, axis);
1444        }
1445
1446        @Override
1447        public boolean any() {
1448                return Comparisons.anyTrue(this);
1449        }
1450
1451        @Override
1452        public BooleanDataset any(final int axis) {
1453                return Comparisons.anyTrue(this, axis);
1454        }
1455
1456        @Override
1457        public Dataset ifloorDivide(final Object o) {
1458                return idivide(o).ifloor();
1459        }
1460
1461        @Override
1462        public double residual(final Object o) {
1463                return residual(o, null, false);
1464        }
1465
1466        @Override
1467        public double residual(final Object o, boolean ignoreNaNs) {
1468                return residual(o, null, ignoreNaNs);
1469        }
1470
1471        /**
1472         * @since 2.0
1473         */
1474        @SuppressWarnings("unchecked")
1475        protected StatisticsMetadata<Number> getStats() {
1476                StatisticsMetadata<Number> md = getFirstMetadata(StatisticsMetadata.class);
1477                if (md == null || md.isDirty(this)) {
1478                        md = new StatisticsMetadataImpl<Number>();
1479                        md.initialize(this);
1480                        setMetadata(md);
1481                }
1482                return md;
1483        }
1484
1485        /**
1486         * @since 2.0
1487         */
1488        @SuppressWarnings("unchecked")
1489        protected StatisticsMetadata<String> getStringStats() {
1490                StatisticsMetadata<String> md = getFirstMetadata(StatisticsMetadata.class);
1491                if (md == null || md.isDirty(this)) {
1492                        md = new StatisticsMetadataImpl<String>();
1493                        md.initialize(this);
1494                        setMetadata(md);
1495                }
1496                return md;
1497        }
1498
1499        @Override
1500        public Number max(boolean... ignoreInvalids) {
1501                return getStats().getMaximum(ignoreInvalids);
1502        }
1503
1504        @Override
1505        public Dataset max(int axis, boolean... ignoreInvalids) {
1506                return getStats().getMaximum(axis, ignoreInvalids);
1507        }
1508
1509        @Override
1510        public Dataset max(int[] axes, boolean... ignoreInvalids) {
1511                return getStats().getMaximum(axes, ignoreInvalids);
1512        }
1513
1514        @Override
1515        public Number min(boolean... ignoreInvalids) {
1516                return getStats().getMinimum(ignoreInvalids);
1517        }
1518
1519        @Override
1520        public Dataset min(int axis, boolean... ignoreInvalids) {
1521                return getStats().getMinimum(axis, ignoreInvalids);
1522        }
1523
1524        @Override
1525        public Dataset min(int[] axes, boolean... ignoreInvalids) {
1526                return getStats().getMinimum(axes, ignoreInvalids);
1527        }
1528
1529        @Override
1530        public int argMax(boolean... ignoreInvalids) {
1531                return getFlat1DIndex(maxPos(ignoreInvalids));
1532        }
1533
1534        /**
1535         * @since 2.0
1536         */
1537        @Override
1538        public IntegerDataset argMax(int axis, boolean... ignoreInvalids) {
1539                return (IntegerDataset) getStats().getArgMaximum(axis, ignoreInvalids);
1540        }
1541
1542        @Override
1543        public int argMin(boolean... ignoreInvalids) {
1544                return getFlat1DIndex(minPos(ignoreInvalids));
1545        }
1546
1547        /**
1548         * @since 2.0
1549         */
1550        @Override
1551        public IntegerDataset argMin(int axis, boolean... ignoreInvalids) {
1552                return (IntegerDataset) getStats().getArgMinimum(axis, ignoreInvalids);
1553        }
1554
1555        @Override
1556        public Number peakToPeak(boolean... ignoreInvalids) {
1557                return DTypeUtils.fromDoubleToBiggestNumber(max(ignoreInvalids).doubleValue() - min(ignoreInvalids).doubleValue(), getDType());
1558        }
1559
1560        @Override
1561        public Dataset peakToPeak(int axis,  boolean... ignoreInvalids) {
1562                return Maths.subtract(max(axis, ignoreInvalids), min(axis, ignoreInvalids));
1563        }
1564
1565        @Override
1566        public Dataset peakToPeak(int[] axes,  boolean... ignoreInvalids) {
1567                return Maths.subtract(max(axes, ignoreInvalids), min(axes, ignoreInvalids));
1568        }
1569
1570        @Override
1571        public long count(boolean... ignoreInvalids) {
1572                return getStats().getCount(ignoreInvalids);
1573        }
1574
1575        @Override
1576        public Dataset count(int axis, boolean... ignoreInvalids) {
1577                return getStats().getCount(axis, ignoreInvalids);
1578        }
1579
1580        @Override
1581        public Dataset count(int[] axes, boolean... ignoreInvalids) {
1582                return getStats().getCount(axes, ignoreInvalids);
1583        }
1584
1585        @Override
1586        public Object sum(boolean... ignoreInvalids) {
1587                return getStats().getSum(ignoreInvalids);
1588        }
1589
1590        @Override
1591        public Dataset sum(int axis, boolean... ignoreInvalids) {
1592                return getStats().getSum(axis, ignoreInvalids);
1593        }
1594
1595        @Override
1596        public Dataset sum(int[] axes, boolean... ignoreInvalids) {
1597                return getStats().getSum(axes, ignoreInvalids);
1598        }
1599
1600        @Override
1601        public Object product(boolean... ignoreInvalids) {
1602                return Stats.product(this, ignoreInvalids);
1603        }
1604
1605        @Override
1606        public Dataset product(int axis, boolean... ignoreInvalids) {
1607                return Stats.product(this, axis, ignoreInvalids);
1608        }
1609
1610        @Override
1611        public Dataset product(int[] axes, boolean... ignoreInvalids) {
1612                return Stats.product(this, axes, ignoreInvalids);
1613        }
1614
1615        @Override
1616        public Object mean(boolean... ignoreInvalids) {
1617                return getStats().getMean(ignoreInvalids);
1618        }
1619
1620        @Override
1621        public Dataset mean(int axis, boolean... ignoreInvalids) {
1622                return getStats().getMean(axis, ignoreInvalids);
1623        }
1624
1625        @Override
1626        public Dataset mean(int[] axes, boolean... ignoreInvalids) {
1627                return getStats().getMean(axes, ignoreInvalids);
1628        }
1629
1630        @Override
1631        public double variance() {
1632                return variance(false);
1633        }
1634
1635        @Override
1636        public double variance(boolean isWholePopulation, boolean... ignoreInvalids) {
1637                return getStats().getVariance(isWholePopulation, ignoreInvalids);
1638        }
1639
1640        @Override
1641        public Dataset variance(int axis) {
1642                return getStats().getVariance(axis, false);
1643        }
1644
1645        @Override
1646        public Dataset variance(int[] axes) {
1647                return getStats().getVariance(axes, false);
1648        }
1649
1650        @Override
1651        public Dataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
1652                return getStats().getVariance(axis, isWholePopulation, ignoreInvalids);
1653        }
1654
1655        @Override
1656        public Dataset variance(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) {
1657                return getStats().getVariance(axes, isWholePopulation, ignoreInvalids);
1658        }
1659
1660        @Override
1661        public double stdDeviation() {
1662                return Math.sqrt(variance());
1663        }
1664
1665        @Override
1666        public double stdDeviation(boolean isWholePopulation, boolean... ignoreInvalids) {
1667                return Math.sqrt(variance(isWholePopulation, ignoreInvalids));
1668        }
1669
1670        @Override
1671        public Dataset stdDeviation(int axis) {
1672                return Maths.sqrt(variance(axis, false));
1673        }
1674
1675        @Override
1676        public Dataset stdDeviation(int[] axes) {
1677                return Maths.sqrt(variance(axes, false));
1678        }
1679
1680        @Override
1681        public Dataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
1682                return Maths.sqrt(variance(axis, isWholePopulation, ignoreInvalids));
1683        }
1684
1685        @Override
1686        public Dataset stdDeviation(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) {
1687                return Maths.sqrt(variance(axes, isWholePopulation, ignoreInvalids));
1688        }
1689
1690        @Override
1691        public double rootMeanSquare(boolean... ignoreInvalids) {
1692                StatisticsMetadata<Number> stats = getStats();
1693                final double mean = stats.getMean(ignoreInvalids).doubleValue();
1694                final double var = stats.getVariance(true, ignoreInvalids);
1695                return Math.sqrt(var + mean * mean);
1696        }
1697
1698        @Override
1699        public Dataset rootMeanSquare(int axis, boolean... ignoreInvalids) {
1700                StatisticsMetadata<Number> stats = getStats();
1701                Dataset v = stats.getVariance(axis, true, ignoreInvalids);
1702                Dataset m = stats.getMean(axis, ignoreInvalids);
1703                Dataset result = Maths.multiply(m, m);
1704                return Maths.sqrt(result.iadd(v));
1705        }
1706
1707        @Override
1708        public Dataset rootMeanSquare(int[] axes, boolean... ignoreInvalids) {
1709                StatisticsMetadata<Number> stats = getStats();
1710                Dataset v = stats.getVariance(axes, true, ignoreInvalids);
1711                Dataset m = stats.getMean(axes, ignoreInvalids);
1712                Dataset result = Maths.multiply(m, m);
1713                return Maths.sqrt(result.iadd(v));
1714        }
1715
1716        /**
1717         * Set item from compatible dataset in a direct and speedy way. Remember to setDirty afterwards.
1718         * 
1719         * @param dindex
1720         * @param sindex
1721         * @param src
1722         *            is the source data buffer
1723         */
1724        protected abstract void setItemDirect(final int dindex, final int sindex, final Object src);
1725
1726        /**
1727         * @return error broadcasted to current shape
1728         */
1729        private Dataset getBroadcastedInternalError() {
1730                if (shape == null) {
1731                        throw new IllegalArgumentException("Cannot get error for null dataset");
1732                }
1733                ILazyDataset led = super.getErrors();
1734                if (led == null)
1735                        return null;
1736
1737                Dataset ed = null;
1738                try {
1739                        ed = DatasetUtils.sliceAndConvertLazyDataset(led);
1740                } catch (DatasetException e) {
1741                        logger.error("Could not get data from lazy dataset", e);
1742                }
1743                if (led != ed) {
1744                        setErrors(ed); // set back
1745                }
1746
1747                return ed.getBroadcastView(shape);
1748        }
1749
1750        @Override
1751        public Dataset getErrors() {
1752                Dataset ed = getBroadcastedInternalError();
1753                if (ed == null)
1754                        return null;
1755
1756                return ed;
1757        }
1758
1759        @Override
1760        public double getError() {
1761                Dataset ed = getBroadcastedInternalError();
1762                if (ed == null)
1763                        return 0;
1764
1765                return ed.getDouble();
1766        }
1767
1768        @Override
1769        public double getError(final int i) {
1770                Dataset ed = getBroadcastedInternalError();
1771                if (ed == null)
1772                        return 0;
1773
1774                return ed.getDouble(i);
1775        }
1776
1777        @Override
1778        public double getError(final int i, final int j) {
1779                Dataset ed = getBroadcastedInternalError();
1780                if (ed == null)
1781                        return 0;
1782
1783                return ed.getDouble(i, j);
1784        }
1785
1786        @Override
1787        public double getError(int... pos) {
1788                Dataset ed = getBroadcastedInternalError();
1789                if (ed == null)
1790                        return 0;
1791
1792                return ed.getDouble(pos);
1793        }
1794
1795        @Override
1796        public double[] getErrorArray(final int i) {
1797                Dataset ed = getBroadcastedInternalError();
1798                if (ed == null)
1799                        return null;
1800
1801                return new double[] {getError(i)};
1802        }
1803
1804        @Override
1805        public double[] getErrorArray(final int i, final int j) {
1806                Dataset ed = getBroadcastedInternalError();
1807                if (ed == null)
1808                        return null;
1809
1810                return new double[] {getError(i, j)};
1811        }
1812
1813        @Override
1814        public double[] getErrorArray(int... pos) {
1815                Dataset ed = getBroadcastedInternalError();
1816                if (ed == null)
1817                        return null;
1818
1819                return new double[] {getError(pos)};
1820        }
1821
1822        protected Dataset getInternalSquaredError() {
1823                Dataset sed = getErrorBuffer().getBroadcastView(shape);
1824                return sed;
1825        }
1826
1827        @Override
1828        public Dataset getErrorBuffer() {
1829                ErrorMetadata emd = getErrorMetadata();
1830                if (emd == null)
1831                        return null;
1832
1833                if (!(emd instanceof ErrorMetadataImpl)) {
1834                        ILazyDataset led = emd.getError();
1835                        Dataset ed;
1836                        try {
1837                                ed = DatasetUtils.sliceAndConvertLazyDataset(led);
1838                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1839                                setMetadata(emd);
1840                                emd.setError(ed);
1841                        } catch (MetadataException me) {
1842                                logger.error("Could not create metadata", me);
1843                        } catch (DatasetException e) {
1844                                logger.error("Could not get data from lazy dataset", e);
1845                        }
1846                }
1847
1848                return ((ErrorMetadataImpl) emd).getSquaredError();
1849        }
1850
1851        /**
1852         * Set a copy of the buffer that backs the (squared) error data
1853         * @param buffer can be null, anything that can be used to create a DoubleDataset or CompoundDoubleDataset
1854         */
1855        @Override
1856        public void setErrorBuffer(Serializable buffer) {
1857                if (shape == null) {
1858                        throw new IllegalArgumentException("Cannot set error buffer for null dataset");
1859                }
1860                if (buffer == null) {
1861                        clearMetadata(ErrorMetadata.class);
1862                        return;
1863                }
1864
1865                IDataset d = (IDataset) createFromSerializable(buffer, false);
1866                ErrorMetadata emd = getErrorMetadata();
1867                if (!(emd instanceof ErrorMetadataImpl)) {
1868                        try {
1869                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1870                                setMetadata(emd);
1871                        } catch (MetadataException me) {
1872                                logger.error("Could not create metadata", me);
1873                        }
1874                }
1875                ((ErrorMetadataImpl) emd).setSquaredError(d);
1876        }
1877}