001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.io.Serializable;
016import java.util.ArrayList;
017import java.util.List;
018
019import org.eclipse.january.DatasetException;
020import org.eclipse.january.IMonitor;
021
022/**
023 * Class to aggregate a set of lazy datasets and present them as a single lazy dataset where
024 * the first position value accesses the aggregation
025 */
026public class AggregateDataset extends LazyDatasetBase implements ILazyDataset {
027
028        /**
029         * Update this when there are any serious changes to API
030         */
031        private static final long serialVersionUID = -5523566223386837581L;
032
033        private ILazyDataset[] data = null; // array of lazy datasets
034        private int[] map = null;    // map first dimension to index of dataset
035        private int[] offset = null; // cumulative first dimension lengths used as slice offsets
036        private int size;
037        private Class<? extends Dataset> clazz = null;
038        private int isize; // number of elements per item
039        protected AggregateDataset base = null;
040        private int[] sliceStart = null;
041        private int[] sliceStep  = null;
042
043        /**
044         * Calculate (possibly extended) shapes from given datasets
045         * @param extend if true, extend rank by one
046         * @param datasets inputs
047         * @return array of shapes
048         */
049        public static int[][] calcShapes(boolean extend, ILazyDataset... datasets) {
050                if (datasets.length == 0)
051                        throw new IllegalArgumentException("No datasets given");
052
053                int maxRank = -1;
054                for (ILazyDataset d : datasets) {
055                        if (d == null)
056                                throw new IllegalArgumentException("Null dataset given");
057
058                        int r = d.getRank();
059                        if (r > maxRank)
060                                maxRank = r;
061                }
062
063                if (extend)
064                        maxRank++;
065
066                int[][] shapes = new int[datasets.length][];
067                for (int j = 0; j < datasets.length; j++) {
068                        ILazyDataset d = datasets[j];
069                        int[] s = d.getShape();
070                        if (s.length < maxRank) {
071                                int[] ns = new int[maxRank];
072                                int start = maxRank - s.length;
073
074                                for (int i = 0; i < start; i++) { // prepend ones as necessary
075                                        ns[i] = 1;
076                                }
077                                for (int i = 0; i < s.length; i++) {
078                                        ns[i+start] = s[i];
079                                }
080                                s = ns;
081                        }
082                        shapes[j] = s;
083                }
084
085                return shapes;
086        }
087
088        AggregateDataset(int itemSize, Class<? extends Dataset> clazz, int... shape) {
089                isize = itemSize;
090                this.shape = shape.clone();
091                try {
092                        size = ShapeUtils.calcSize(shape);
093                } catch (IllegalArgumentException e) {
094                        size = Integer.MAX_VALUE; // this indicates that the entire dataset cannot be read in! 
095                }
096                this.clazz = clazz;
097        }
098
099        AggregateDataset(int itemSize, int[] shape, int dtype) {
100                this(itemSize, DTypeUtils.getInterface(dtype), shape);
101        }
102
103        /**
104         * Create an aggregate dataset
105         * @param extend if true, extend rank by one
106         * @param datasets inputs
107         */
108        public AggregateDataset(boolean extend, ILazyDataset... datasets) {
109
110                final int[][] shapes = calcShapes(extend, datasets);
111
112                // check for same (sub-)shape
113                final int[] s = shapes[0];
114                final int axis = extend ? -1 : 0;
115                for (int j = 1; j < shapes.length; j++) {
116                        if (!ShapeUtils.areShapesCompatible(s, shapes[j], axis))
117                                throw new IllegalArgumentException("Dataset '" + datasets[j].getName() + "' has wrong shape");
118                }
119
120                // set shapes of datasets
121                final int maxRank = s.length;
122                data = new ILazyDataset[datasets.length];
123                isize = datasets[0].getElementsPerItem();
124                for (int j = 0; j < datasets.length; j++) {
125                        ILazyDataset d = datasets[j];
126                        int[] ds = d.getShape();
127                        if (ds.length < maxRank) {
128                                d = d.clone();
129                                d.setShape(shapes[j]);
130                        }
131                        data[j] = d;
132                        if (d.getElementsPerItem() != isize) {
133                                throw new IllegalArgumentException("All datasets must have the same number of elements");
134                        }
135                }
136
137                // calculate new shape
138                shape = new int[maxRank];
139                for (int i = 1; i < shape.length; i++) {
140                        shape[i] = s[i];
141                }
142                if (extend) {
143                        shape[0] = data.length;
144                } else {
145                        for (int j = 0; j < datasets.length; j++) {
146                                shape[0] += shapes[j][0];
147                        }
148                }
149                try {
150                        size = ShapeUtils.calcSize(shape);
151                } catch (IllegalArgumentException e) {
152                        size = Integer.MAX_VALUE; // this indicates that the entire dataset cannot be read in! 
153                }
154
155                // work out offsets from cumulative lengths
156                offset = new int[data.length];
157                int cd = 0;
158                for (int i = 0; i < data.length; i++) {
159                        offset[i] = cd;
160                        cd += data[i].getShape()[0];
161                }
162
163                // calculate mapping from aggregate dimension to dataset array index
164                map = new int[shape[0]];
165                int k = 0;
166                for (int i = 0; i < data.length; i++) {
167                        int jmax = data[i].getShape()[0];
168                        for (int j = 0; j < jmax; j++)
169                                map[k++] = i;
170                }
171
172                for (ILazyDataset d : data) {
173                        clazz = InterfaceUtils.getBestInterface(clazz, InterfaceUtils.getInterfaceFromClass(d.getElementsPerItem(), d.getElementClass()));
174                }
175
176                for (ILazyDataset d : data) {
177                        String n = d.getName();
178                        if (n != null) {
179                                name = n;
180                                break;
181                        }
182                }
183        }
184
185        @Override
186        public Class<?> getElementClass() {
187                return InterfaceUtils.getElementClass(clazz);
188        }
189
190        @Override
191        public int getElementsPerItem() {
192                return isize;
193        }
194
195        @Override
196        public int getDType() {
197                return DTypeUtils.getDType(clazz);
198        }
199
200        @Override
201        public int getSize() {
202                return size;
203        }
204
205        @Override
206        public void setShape(int... shape) {
207                throw new UnsupportedOperationException("Not implemented");
208        }
209
210        @Override
211        public ILazyDataset squeezeEnds() {
212                throw new UnsupportedOperationException("Not implemented");
213        }
214
215        @Override
216        public Dataset getSlice(int[] start, int[] stop, int[] step) throws DatasetException {
217                return getSlice(null, new SliceND(shape, start, stop, step));
218        }
219
220        @Override
221        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) throws DatasetException {
222                return getSlice(monitor, new SliceND(shape, start, stop, step));
223        }
224
225        @Override
226        public Dataset getSlice(IMonitor monitor, SliceND slice) throws DatasetException {
227                int[] start = slice.getStart();
228                int[] stop  = slice.getStop();
229                int[] step  = slice.getStep();
230
231                if (base != null) {
232                        for (int i = 0; i < shape.length; i++) {
233                                start[i] = sliceStart[i] + start[i] * sliceStep[i];
234                                stop[i]  = sliceStart[i] + (stop[i] - 1) * sliceStep[i] + 1;
235                                step[i]  = step[i] * sliceStep[i];
236                        }
237                        return base.getSlice(monitor, start, stop, step);
238                }
239
240                // convert first dimension's slice to individual slices per stored dataset
241                int fb = start[0];
242                int fe = stop[0];
243                int fs = step[0];
244
245                List<Dataset> sliced = new ArrayList<Dataset>();
246                int op = fb;
247                int p = op;
248                ILazyDataset od = data[map[op]];
249                ILazyDataset nd; 
250                while (p < fe) {
251                        
252                        if (monitor!=null && monitor.isCancelled()) throw new DatasetException("Slice cancelled");
253                        nd = data[map[p]];
254                        if (nd != od) {
255                                start[0] = op - offset[map[op]];
256                                stop[0] = p - offset[map[op]];
257                                Dataset a = DatasetUtils.convertToDataset(od.getSlice(monitor, start, stop, step));
258                                sliced.add(a.cast(clazz));
259
260                                od = nd;
261                                op = p;
262                        }
263                        p += fs;
264                }
265                start[0] = op - offset[map[op]];
266                stop[0] = p - offset[map[op]];
267                Dataset a = DatasetUtils.convertToDataset(od.getSlice(monitor, start, stop, step));
268                sliced.add(a.cast(clazz));
269
270                Dataset d = DatasetUtils.concatenate(sliced.toArray(new Dataset[0]), 0);
271                d.setName(name);
272                return d;
273        }
274
275        @Override
276        public Dataset getSlice(Slice... slice) throws DatasetException {
277                return getSlice(null, slice);
278        }
279
280        @Override
281        public Dataset getSlice(SliceND slice) throws DatasetException {
282                return getSlice(null, slice);
283        }
284
285        @Override
286        public Dataset getSlice(IMonitor monitor, Slice... slice) throws DatasetException {
287                return getSlice(monitor, new SliceND(shape, slice));
288        }
289
290        @Override
291        public AggregateDataset getSliceView(Slice... slice) {
292                return getSliceView(new SliceND(shape, slice));
293        }
294
295        @Override
296        public AggregateDataset getSliceView(int[] start, int[] stop, int[] step) {
297                return getSliceView(new SliceND(shape, start, stop, step));
298        }
299
300        @Override
301        public AggregateDataset getSliceView(SliceND slice) {
302                AggregateDataset lazy = new AggregateDataset(isize, clazz, slice.getShape());
303                lazy.sliceStart = slice.getStart();
304                lazy.sliceStep  = slice.getStep();
305                lazy.name = name + "[" + slice + "]";
306                lazy.base = base == null ? this : base;
307                return lazy;
308        }
309
310        @Override
311        public AggregateDataset getTransposedView(int... axes) {
312                throw new UnsupportedOperationException("Not implemented");
313        }
314
315        @Override
316        public AggregateDataset clone() {
317                throw new UnsupportedOperationException("Not implemented");
318        }
319
320        @Override
321        public String toString() {
322                StringBuilder out = new StringBuilder();
323
324                if (name != null && name.length() > 0) {
325                        out.append("Aggregate dataset '");
326                        out.append(name);
327                        out.append("' has shape [");
328                } else {
329                        out.append("Aggregate dataset shape is [");
330                }
331                int rank = shape == null ? 0 : shape.length;
332
333                if (rank > 0 && shape[0] > 0) {
334                        out.append(shape[0]);
335                }
336                for (int i = 1; i < rank; i++) {
337                        out.append(", " + shape[i]);
338                }
339                out.append(']');
340
341                return out.toString();
342        }
343        
344        @Override
345        public void setErrors(Serializable errors) {
346                throw new RuntimeException("setLazyErrors is unimplemented for "+getClass().getSimpleName());
347        }
348        
349        @Override
350        public ILazyDataset getErrors() {
351                return null;
352        }
353
354}