001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.IOException; 016import java.io.Serializable; 017import java.lang.annotation.Annotation; 018import java.lang.reflect.Field; 019import java.util.ArrayList; 020import java.util.Arrays; 021import java.util.HashMap; 022import java.util.LinkedList; 023import java.util.List; 024import java.util.Map; 025import java.util.Objects; 026 027import org.eclipse.january.DatasetException; 028import org.eclipse.january.IMonitor; 029import org.eclipse.january.io.ILazyLoader; 030import org.eclipse.january.metadata.MetadataFactory; 031import org.eclipse.january.metadata.MetadataType; 032import org.eclipse.january.metadata.OriginMetadata; 033import org.eclipse.january.metadata.Reshapeable; 034import org.eclipse.january.metadata.Sliceable; 035import org.eclipse.january.metadata.Transposable; 036import org.slf4j.Logger; 037import org.slf4j.LoggerFactory; 038 039public class LazyDataset extends LazyDatasetBase implements Serializable, Cloneable { 040 private static final long serialVersionUID = 2467865859867440242L; 041 042 private static final Logger logger = LoggerFactory.getLogger(LazyDataset.class); 043 044 protected Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata = null; 045 protected int[] oShape; // original shape 046 protected long size; // number of items 047 private Class<? extends Dataset> clazz = null; 048 protected int isize; // number of elements per item 049 050 protected ILazyLoader loader; 051 052 // relative to loader 053 protected int[] begSlice = null; // slice begin 054 protected int[] delSlice = null; // slice delta 055 /** 056 * @since 2.2 057 */ 058 protected int[] sShape = null; // sliced shape 059 060 /** 061 * @since 2.2 062 */ 063 protected int[] padding = null; // differences in shape from original (or sliced) shape 064 protected int[] map; // transposition map (same length as current shape) 065 066 /** 067 * Create a lazy dataset 068 * @param loader lazy loader 069 * @param name of dataset 070 * @param elements item size 071 * @param clazz dataset sub-interface 072 * @param shape dataset shape 073 * @since 2.3 074 */ 075 public LazyDataset(ILazyLoader loader, String name, int elements, Class<? extends Dataset> clazz, int... shape) { 076 this.loader = loader; 077 this.name = name; 078 this.isize = elements; 079 this.clazz = clazz; 080 this.shape = shape.clone(); 081 this.oShape = this.shape; 082 try { 083 size = ShapeUtils.calcLongSize(shape); 084 } catch (IllegalArgumentException e) { 085 size = Long.MAX_VALUE; // this indicates that the entire dataset cannot be read in! 086 } 087 } 088 089 /** 090 * Create a lazy dataset 091 * @param loader lazy loader 092 * @param name of dataset 093 * @param clazz dataset sub-interface 094 * @param shape dataset shape 095 * @since 2.3 096 */ 097 public LazyDataset(ILazyLoader loader, String name, Class<? extends Dataset> clazz, int... shape) { 098 this(loader, name, 1, clazz, shape); 099 } 100 101 /** 102 * Create a lazy dataset 103 * @param name of dataset 104 * @param dtype dataset type 105 * @param elements item size 106 * @param shape dataset shape 107 * @param loader lazy loader 108 * @deprecated Use {@link #LazyDataset(ILazyLoader, String, int, Class, int[])} 109 */ 110 @Deprecated 111 public LazyDataset(String name, int dtype, int elements, int[] shape, ILazyLoader loader) { 112 this(loader, name, elements, DTypeUtils.getInterface(dtype), shape); 113 } 114 115 /** 116 * Create a lazy dataset 117 * @param name of dataset 118 * @param dtype dataset type 119 * @param shape dataset shape 120 * @param loader lazy loader 121 * @deprecated Use {@link #LazyDataset(ILazyLoader, String, int, Class, int[])} 122 */ 123 @Deprecated 124 public LazyDataset(String name, int dtype, int[] shape, ILazyLoader loader) { 125 this(name, dtype, 1, shape, loader); 126 } 127 128 LazyDataset(LazyDataset other) { 129 name = other.name; 130 shape = other.shape.clone(); 131 metadata = other.copyMetadata(); 132 oMetadata = other.oMetadata; 133 oShape = other.oShape; 134 size = other.size; 135 clazz = other.clazz; 136 isize = other.isize; 137 loader = other.loader; 138 139 begSlice = other.begSlice; 140 delSlice = other.delSlice; 141 sShape = other.sShape; 142 padding = other.padding; 143 map = other.map; 144 } 145 146 /** 147 * Create a lazy dataset based on in-memory data (handy for testing) 148 * @param dataset input 149 * @return lazy dataset 150 */ 151 public static LazyDataset createLazyDataset(final Dataset dataset) { 152 return new LazyDataset(new ILazyLoader() { 153 private static final long serialVersionUID = -6725268922780517523L; 154 155 final Dataset d = dataset; 156 157 @Override 158 public boolean isFileReadable() { 159 return true; 160 } 161 162 @Override 163 public Dataset getDataset(IMonitor mon, SliceND slice) throws IOException { 164 return d.getSlice(mon, slice); 165 } 166 }, dataset.getName(), dataset.getElementsPerItem(), dataset.getClass(), dataset.getShapeRef()); 167 } 168 169 @Override 170 public Class<?> getElementClass() { 171 return InterfaceUtils.getElementClass(clazz); 172 } 173 174 /** 175 * Can return -1 when dataset interface is not defined 176 */ 177 @Override 178 public int getDType() { 179 return clazz == null ? -1 : DTypeUtils.getDType(clazz); 180 } 181 182 /** 183 * @return dataset interface that supports element class and number of elements. Can be null when undefined 184 * @since 2.3 185 */ 186 public Class<? extends Dataset> getInterface() { 187 return clazz; 188 } 189 190 /** 191 * Set interface 192 * @param clazz dataset sub-interface 193 * @since 2.3 194 */ 195 public void setInterface(Class<? extends Dataset> clazz) { 196 this.clazz = clazz; 197 } 198 199 /** 200 * Can return -1 for unknown 201 */ 202 @Override 203 public int getElementsPerItem() { 204 return isize; 205 } 206 207 @Override 208 public int getSize() { 209 return (int) size; 210 } 211 212 @Override 213 public String toString() { 214 StringBuilder out = new StringBuilder(); 215 216 if (name != null && name.length() > 0) { 217 out.append("Lazy dataset '"); 218 out.append(name); 219 out.append("' has shape ["); 220 } else { 221 out.append("Lazy dataset shape is ["); 222 } 223 int rank = shape == null ? 0 : shape.length; 224 225 if (rank > 0 && shape[0] >= 0) { 226 out.append(shape[0]); 227 } 228 for (int i = 1; i < rank; i++) { 229 out.append(", " + shape[i]); 230 } 231 out.append(']'); 232 233 return out.toString(); 234 } 235 236 @Override 237 public int hashCode() { 238 final int prime = 31; 239 int result = super.hashCode(); 240 result = prime * result + Arrays.hashCode(oShape); 241 result = prime * result + (int) (size ^ (size >>> 32)); 242 result = prime * result + Objects.hashCode(clazz); 243 result = prime * result + isize; 244 result = prime * result + Objects.hashCode(loader); 245 result = prime * result + Arrays.hashCode(begSlice); 246 result = prime * result + Arrays.hashCode(delSlice); 247 result = prime * result + Arrays.hashCode(sShape); 248 result = prime * result + Arrays.hashCode(padding); 249 result = prime * result + Arrays.hashCode(map); 250 return result; 251 } 252 253 @Override 254 public boolean equals(Object obj) { 255 if (this == obj) { 256 return true; 257 } 258 if (!super.equals(obj)) { 259 return false; 260 } 261 262 LazyDataset other = (LazyDataset) obj; 263 if (!Arrays.equals(oShape, other.oShape)) { 264 return false; 265 } 266 if (size != other.size) { 267 return false; 268 } 269 if (!Objects.equals(clazz, other.clazz)) { 270 return false; 271 } 272 if (isize != other.isize) { 273 return false; 274 } 275 276 if (loader != other.loader) { 277 return false; 278 } 279 280 if (!Arrays.equals(begSlice, other.begSlice)) { 281 return false; 282 } 283 if (!Arrays.equals(delSlice, other.delSlice)) { 284 return false; 285 } 286 if (!Arrays.equals(sShape, other.sShape)) { 287 return false; 288 } 289 if (!Arrays.equals(padding, other.padding)) { 290 return false; 291 } 292 if (!Arrays.equals(map, other.map)) { 293 return false; 294 } 295 296 return true; 297 } 298 299 @Override 300 public LazyDataset clone() { 301 return new LazyDataset(this); 302 } 303 304 @Override 305 public void setShape(int... shape) { 306 setShapeInternal(shape.clone()); 307 } 308 309 @Override 310 public LazyDataset squeezeEnds() { 311 setShapeInternal(ShapeUtils.squeezeShape(shape, true)); 312 return this; 313 } 314 315 @Override 316 public Dataset getSlice(int[] start, int[] stop, int[] step) throws DatasetException { 317 return getSlice(null, start, stop, step); 318 } 319 320 @Override 321 public Dataset getSlice(Slice... slice) throws DatasetException { 322 if (slice == null || slice.length == 0) { 323 return internalGetSlice(null, new SliceND(shape)); 324 } 325 return internalGetSlice(null, new SliceND(shape, slice)); 326 } 327 328 @Override 329 public Dataset getSlice(SliceND slice) throws DatasetException { 330 return getSlice(null, slice); 331 } 332 333 @Override 334 public Dataset getSlice(IMonitor monitor, Slice... slice) throws DatasetException { 335 if (slice == null || slice.length == 0) { 336 return internalGetSlice(monitor, new SliceND(shape)); 337 } 338 return internalGetSlice(monitor, new SliceND(shape, slice)); 339 } 340 341 @Override 342 public LazyDataset getSliceView(Slice... slice) { 343 if (slice == null || slice.length == 0) { 344 return getSliceView(new SliceND(shape)); 345 } 346 return getSliceView(new SliceND(shape, slice)); 347 } 348 349 /** 350 * @param nShape 351 */ 352 private void setShapeInternal(int... nShape) { 353 // work out transposed (sliced) shape (instead of removing padding from current shape) 354 if (size != 0) { 355 int[] pShape = calcTransposed(map, sShape == null ? oShape : sShape); 356 padding = ShapeUtils.calcShapePadding(pShape, nShape); 357 } 358 359 if (metadata != null) { 360 storeMetadata(metadata, Reshapeable.class); 361 metadata = copyMetadata(); 362 reshapeMetadata(shape, nShape); 363 } 364 shape = nShape; 365 } 366 367 @Override 368 public LazyDataset getSliceView(int[] start, int[] stop, int[] step) { 369 return internalGetSliceView(new SliceND(shape, start, stop, step)); 370 } 371 372 @Override 373 public LazyDataset getSliceView(SliceND slice) { 374 checkSliceND(slice); 375 return internalGetSliceView(slice); 376 } 377 378 protected LazyDataset internalGetSliceView(SliceND slice) { 379 LazyDataset view = clone(); 380 if (slice == null || slice.isAll()) { 381 return view; 382 } 383 384 SliceND nslice = calcTrueSlice(slice); 385 if (nslice != null) { 386 view.begSlice = nslice.getStart(); 387 view.delSlice = nslice.getStep(); 388 view.sShape = nslice.getShape(); 389 } 390 view.shape = slice.getShape(); 391 view.size = ShapeUtils.calcLongSize(view.shape); 392 view.storeMetadata(metadata, Sliceable.class); 393 394 view.sliceMetadata(true, slice); 395 return view; 396 } 397 398 @Override 399 public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) throws DatasetException { 400 return internalGetSlice(monitor, new SliceND(shape, start, stop, step)); 401 } 402 403 @Override 404 public Dataset getSlice(IMonitor monitor, SliceND slice) throws DatasetException { 405 checkSliceND(slice); 406 407 return internalGetSlice(monitor, slice); 408 } 409 410 protected Dataset internalGetSlice(IMonitor monitor, SliceND slice) throws DatasetException { 411 if (loader != null && !loader.isFileReadable()) { 412 return null; 413 } 414 415 SliceND nslice = calcTrueSlice(slice); 416 417 Dataset a; 418 if (nslice == null) { 419 a = DatasetFactory.zeros(clazz == null ? DoubleDataset.class : clazz, slice == null ? shape : slice.getShape()); 420 } else { 421 try { 422 a = DatasetUtils.convertToDataset(loader.getDataset(monitor, nslice)); 423 } catch (IOException e) { 424 logger.error("Problem getting {}: {}", slice == null ? "all" : String.format("slice %s %s %s from %s", Arrays.toString(slice.getStart()), Arrays.toString(slice.getStop()), 425 Arrays.toString(slice.getStep()), loader), e); 426 throw new DatasetException(e); 427 } 428 } 429 a.setName(name + AbstractDataset.BLOCK_OPEN + (nslice == null ? 430 (slice == null ? "..." : slice) : nslice) + AbstractDataset.BLOCK_CLOSE); 431 if (metadata != null && a instanceof LazyDatasetBase) { 432 LazyDatasetBase ba = (LazyDatasetBase) a; 433 ba.metadata = copyMetadata(); 434 if (oMetadata != null) { 435 ba.restoreMetadata(oMetadata); 436 } 437 // metadata axis may be larger than data 438 if (nslice != null && (!nslice.isAll() || nslice.getMaxShape() != nslice.getShape())) { 439 ba.sliceMetadata(true, nslice); 440 } 441 } 442 443 if (nslice != null) { 444 if (map != null) { 445 a = a.getTransposedView(map); 446 } 447 if (padding != null) { 448 a.setShape(slice == null ? shape : slice.getShape()); 449 } 450 } 451 a.addMetadata(MetadataFactory.createMetadata(OriginMetadata.class, this, nslice == null ? 452 (slice == null ? null : slice.convertToSlice()) : nslice.convertToSlice(), oShape, null, name)); 453 454 if (clazz == null) { 455 clazz = a.getClass(); 456 } 457 return a; 458 } 459 460 @Override 461 public LazyDataset getTransposedView(final int... axes) { 462 LazyDataset view = clone(); 463 464 int[] naxes = checkPermutatedAxes(shape, axes); 465 if (naxes == null) { 466 return view; 467 } 468 469 view.shape = calcTransposed(naxes, shape); 470 if (view.size != 0 && padding != null) { // work out transpose by reverting effect of padding 471 int or = oShape.length; 472 int nr = shape.length; 473 int j = 0; // naxes index 474 int[] mShape = calcTransposed(map, sShape == null ? oShape : sShape); // pre-padded shape 475 int m = 0; // shape index 476 int e = -1; // index of unit dimension 477 final List<Integer> uaxes = new LinkedList<>(); 478 for (int a : naxes) { 479 uaxes.add(a); 480 } 481 List<Integer> oList = new ArrayList<>(); // dimensions left out by padding (in order) 482 int np = padding.length; 483 for (int i = 0; i < np; i++) { 484 int p = padding[i]; 485 if (p > 0) { // remove added dimensions 486 for (int k = 0; k < p; k++, j++) { 487 uaxes.remove((Integer) j); 488 } 489 } else if (p == 0) { // leave alone 490 if (mShape[m] == 1) { // bump up last unit dimension index 491 e = m; 492 } 493 j++; 494 m++; 495 } else { // add omitted dimensions to list 496 p = -p; 497 for (int k = 0; k < p; k++) { 498 e = find(mShape, 1, e + 1); 499 oList.add(e); 500 } 501 } 502 } 503 504 int[] omitted = new int[oList.size()]; 505 j = 0; 506 for (Integer o : oList) { 507 omitted[j++] = o; 508 } 509 int[] used = new int[or - omitted.length]; // all dimensions not omitted in pre-padded shape 510 j = 0; 511 for (int i = 0; i < or; i++) { 512 if (Arrays.binarySearch(omitted, i) < 0) { 513 used[j++] = i; 514 } 515 } 516 517 int[] vaxes = new int[uaxes.size()]; 518 j = 0; 519 for (int i = 0; i < nr; i++) { // remap dimension numbering 520 int l = uaxes.indexOf(i); 521 if (l >= 0) { 522 vaxes[l] = used[j++]; 523 } 524 } 525 int[] taxes = new int[or]; 526 j = 0; 527 for (int i = 0; i < or; i++) { // reassemble map 528 if (Arrays.binarySearch(omitted, i) >= 0) { 529 taxes[i] = i; 530 } else { 531 taxes[i] = vaxes[j++]; 532 } 533 } 534 535 naxes = taxes; 536 } 537 538 view.map = map == null ? naxes : calcTransposed(naxes, map); 539 if (view.size != 0) { 540 // work out transposed (sliced) shape 541 int[] tShape = calcTransposed(view.map, sShape == null ? oShape : sShape); 542 try { 543 view.padding = ShapeUtils.calcShapePadding(tShape, view.shape); 544 } catch (IllegalArgumentException e) { 545 System.err.println(e.getMessage() + ": " + Arrays.toString(tShape) + " cf " + Arrays.toString(view.shape)); 546 } 547 } 548 view.storeMetadata(metadata, Transposable.class); 549 view.transposeMetadata(axes); 550 return view; 551 } 552 553 private static int find(int[] map, int m, int off) { 554 for (int i = off, imax = map.length; i < imax; i++) { 555 if (map[i] == m) { 556 return i; 557 } 558 } 559 return -1; 560 } 561 562 private static int[] calcTransposed(int[] map, int[] values) { 563 if (values == null) { 564 return null; 565 } 566 int r = values.length; 567 if (map == null || r < 2) { 568 return values; 569 } 570 int[] ovalues = new int[r]; 571 for (int i = 0; i < r; i++) { 572 ovalues[i] = values[map[i]]; 573 } 574 return ovalues; 575 } 576 577 /** 578 * Calculate absolute slice 579 * @param slice an n-D slice 580 * @return true slice or null if zero-sized 581 */ 582 protected final SliceND calcTrueSlice(SliceND slice) { 583 /* 584 * Lazy dataset operations: getTransposedView (T), getSliceView (G), setShape/squeezeEnds (S+/S-): 585 * 586 * . T sets shape, base, and map in new view 587 * . G sets shape, size, begSlice and delSlice in new view 588 * . S sets shape, shapePadding in current view 589 * 590 * Then getSlice needs to interpret all info to find true slice, load data, get transposition (view) 591 * and set shape. Therefore: 592 * . S needs to update shapePadding only 593 * . T needs to update shapePadding too 594 * . G needs to work out true slice to update 595 * 596 * slice -> true slice 597 * adjusts for shape (S^-1) then remap dimensions (T^-1) 598 */ 599 600 if (slice == null) { 601 slice = new SliceND(shape); 602 } 603 604 if (ShapeUtils.calcLongSize(slice.getShape()) == 0) { 605 return null; 606 } 607 608 int[] nshape; 609 int[] nstart; 610 int[] nstep; 611 612 int r = oShape.length; 613 if (padding == null) { 614 nshape = slice.getShape(); 615 nstart = slice.getStart(); 616 nstep = slice.getStep(); 617 } else { 618 final int[] lshape = slice.getShape(); 619 final int[] lstart = slice.getStart(); 620 final int[] lstep = slice.getStep(); 621 622 nstart = new int[r]; 623 nstep = new int[r]; 624 nshape = new int[r]; 625 int i = 0; 626 int j = 0; 627 for (int p : padding) { // remove padding 628 if (p == 0) { 629 nshape[i] = lshape[j]; 630 nstart[i] = lstart[j]; 631 nstep[i] = lstep[j]; 632 i++; 633 j++; 634 } else if (p < 0) { 635 int imax = i - p; 636 while (i < imax) { 637 nshape[i] = 1; 638 nstep[i] = 1; 639 i++; 640 } 641 } else { 642 j += p; 643 } 644 } 645 } 646 647 if (map != null && r > 1) { // transpose dimensions 648 int[] pshape = new int[r]; 649 int[] pstart = new int[r]; 650 int[] pstep = new int[r]; 651 for (int i = 0; i < r; i++) { 652 int m = map[i]; 653 pshape[m] = nshape[i]; 654 pstart[m] = nstart[i]; 655 pstep[m] = nstep[i]; 656 } 657 658 nshape = pshape; 659 nstart = pstart; 660 nstep = pstep; 661 } 662 663 int[] nstop = new int[r]; 664 if (begSlice != null) { // find net slice 665 for (int i = 0; i < r; i++) { 666 int b = begSlice[i]; 667 int d = delSlice[i]; 668 nstart[i] = b + nstart[i] * d; 669 int nd = nstep[i] * d; 670 nstep[i] = nd; 671 nstop[i] = nstart[i] + (nshape[i] - 1) * nd + (nd >= 0 ? 1 : -1); 672 } 673 } else { 674 for (int i = 0; i < r; i++) { 675 int d = nstep[i]; 676 nstop[i] = nstart[i] + (nshape[i] - 1) * d + (d >= 0 ? 1 : -1); 677 } 678 } 679 680 return createSlice(nstart, nstop, nstep); 681 } 682 683 protected SliceND createSlice(int[] nstart, int[] nstop, int[] nstep) { 684 return SliceND.createSlice(oShape, null, nstart, nstop, nstep); 685 } 686 687 /** 688 * Transform data so that it can be used in setSlice of saver 689 * @param data input 690 * @param tslice true slice 691 * @return data with dimensions adjusted and remapped 692 */ 693 final IDataset transformInput(IDataset data, SliceND tslice) { 694 if (padding != null) { // remove padding 695 data = data.getSliceView(); 696 int[] nshape = tslice == null ? shape : tslice.getShape(); 697 data.setShape(nshape); 698 } 699 700 return map == null ? data : data.getTransposedView(map); 701 } 702 703 /** 704 * Store metadata items that has given annotation 705 * @param origMetadata original metadata 706 * @param aclazz annotation class 707 */ 708 private void storeMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> origMetadata, Class<? extends Annotation> aclazz) { 709 List<Class<? extends MetadataType>> mclazzes = findAnnotatedMetadata(aclazz); 710 if (mclazzes.size() == 0) { 711 return; 712 } 713 714 if (oMetadata == null) { 715 oMetadata = new HashMap<Class<? extends MetadataType>, List<MetadataType>>(); 716 } 717 for (Class<? extends MetadataType> mc : mclazzes) { 718 if (oMetadata.containsKey(mc)) { 719 continue; // do not overwrite original 720 } 721 722 List<MetadataType> l = origMetadata.get(mc); 723 List<MetadataType> nl = new ArrayList<MetadataType>(l.size()); 724 for (MetadataType m : l) { 725 nl.add(m.clone()); 726 } 727 oMetadata.put(mc, nl); 728 } 729 } 730 731 @SuppressWarnings("unchecked") 732 private List<Class<? extends MetadataType>> findAnnotatedMetadata(Class<? extends Annotation> aclazz) { 733 List<Class<? extends MetadataType>> mclazzes = new ArrayList<Class<? extends MetadataType>>(); 734 if (metadata == null) { 735 return mclazzes; 736 } 737 738 for (Class<? extends MetadataType> c : metadata.keySet()) { 739 boolean hasAnn = false; 740 for (MetadataType m : metadata.get(c)) { 741 if (m == null) { 742 continue; 743 } 744 745 Class<? extends MetadataType> mc = m.getClass(); 746 do { // iterate over super-classes 747 for (Field f : mc.getDeclaredFields()) { 748 if (f.isAnnotationPresent(aclazz)) { 749 hasAnn = true; 750 break; 751 } 752 } 753 Class<?> sclazz = mc.getSuperclass(); 754 if (!MetadataType.class.isAssignableFrom(sclazz)) { 755 break; 756 } 757 mc = (Class<? extends MetadataType>) sclazz; 758 } while (!hasAnn); 759 if (hasAnn) { 760 break; 761 } 762 } 763 if (hasAnn) { 764 mclazzes.add(c); 765 } 766 } 767 return mclazzes; 768 } 769 770 /** 771 * Gets the maximum size of a slice of a dataset in a given dimension 772 * which should normally fit in memory. Note that it might be possible 773 * to get more in memory, this is a conservative estimate and seems to 774 * almost always work at the size returned; providing Xmx is less than 775 * the physical memory. 776 * 777 * To get more in memory increase -Xmx setting or use an expression 778 * which calls a rolling function (like rmean) instead of slicing directly 779 * to memory. 780 * 781 * @param lazySet lazy dataset 782 * @param dimension to slice along 783 * @return maximum size of dimension that can be sliced. 784 */ 785 public static int getMaxSliceLength(ILazyDataset lazySet, int dimension) { 786 // size in bytes of each item 787 final double size = InterfaceUtils.getItemBytes(lazySet.getElementsPerItem(), InterfaceUtils.getInterface(lazySet)); 788 789 // Max in bytes takes into account our minimum requirement 790 final double max = Math.max(Runtime.getRuntime().totalMemory(), Runtime.getRuntime().maxMemory()); 791 792 // Firstly if the whole dataset it likely to fit in memory, then we allow it. 793 // Space specified in bytes per item available 794 final double space = max/lazySet.getSize(); 795 796 // If we have room for this whole dataset, then fine 797 int[] shape = lazySet.getShape(); 798 if (space >= size) { 799 return shape[dimension]; 800 } 801 802 // Otherwise estimate what we can fit in, conservatively. 803 // First get size of one slice, see it that fits, if not, still return 1 804 double sizeOneSlice = size; // in bytes 805 for (int dim = 0; dim < shape.length; dim++) { 806 if (dim == dimension) { 807 continue; 808 } 809 sizeOneSlice *= shape[dim]; 810 } 811 double avail = max / sizeOneSlice; 812 if (avail < 1) { 813 return 1; 814 } 815 816 // We fudge this to leave some room 817 return (int) Math.floor(avail/4d); 818 } 819}