001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.Serializable; 016import java.lang.annotation.Annotation; 017import java.lang.reflect.Array; 018import java.lang.reflect.Field; 019import java.util.ArrayList; 020import java.util.Arrays; 021import java.util.Collection; 022import java.util.concurrent.ConcurrentMap; 023import java.util.concurrent.ConcurrentHashMap; 024import java.util.List; 025import java.util.Map; 026 027import org.eclipse.january.DatasetException; 028import org.eclipse.january.MetadataException; 029import org.eclipse.january.metadata.Dirtiable; 030import org.eclipse.january.metadata.ErrorMetadata; 031import org.eclipse.january.metadata.IMetadata; 032import org.eclipse.january.metadata.MetadataFactory; 033import org.eclipse.january.metadata.MetadataType; 034import org.eclipse.january.metadata.Reshapeable; 035import org.eclipse.january.metadata.Sliceable; 036import org.eclipse.january.metadata.Transposable; 037import org.slf4j.Logger; 038import org.slf4j.LoggerFactory; 039 040/** 041 * Common base for both lazy and normal dataset implementations 042 */ 043public abstract class LazyDatasetBase implements ILazyDataset, Serializable { 044 045 private static final long serialVersionUID = 767926846438976050L; 046 047 protected static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class); 048 049 protected static boolean catchExceptions; 050 051 static { 052 /** 053 * Boolean to set to true if running jython scripts that utilise ScisoftPy in IDE 054 */ 055 try { 056 catchExceptions = Boolean.getBoolean("run.in.eclipse"); 057 } catch (SecurityException e) { 058 // set a default for when the security manager does not allow access to the requested key 059 catchExceptions = false; 060 } 061 } 062 063 transient private boolean dirty = true; // indicate dirty state of metadata 064 protected String name = ""; 065 066 /** 067 * The shape or dimensions of the dataset 068 */ 069 protected int[] shape; 070 071 protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null; 072 073 /** 074 * @return type of dataset item 075 */ 076 abstract public int getDType(); 077 078 @Override 079 public Class<?> getElementClass() { 080 return DTypeUtils.getElementClass(getDType()); 081 } 082 083 @Override 084 public LazyDatasetBase clone() { 085 return null; 086 } 087 088 @Override 089 public boolean equals(Object obj) { 090 if (this == obj) { 091 return true; 092 } 093 if (obj == null) { 094 return false; 095 } 096 if (!getClass().equals(obj.getClass())) { 097 return false; 098 } 099 100 LazyDatasetBase other = (LazyDatasetBase) obj; 101 if (getDType() != other.getDType()) { 102 return false; 103 } 104 if (getElementsPerItem() != other.getElementsPerItem()) { 105 return false; 106 } 107 if (!Arrays.equals(shape, other.shape)) { 108 return false; 109 } 110 return true; 111 } 112 113 @Override 114 public int hashCode() { 115 int hash = getDType() * 17 + getElementsPerItem(); 116 int rank = shape.length; 117 for (int i = 0; i < rank; i++) { 118 hash = hash*17 + shape[i]; 119 } 120 return hash; 121 } 122 123 @Override 124 public String getName() { 125 return name; 126 } 127 128 @Override 129 public void setName(String name) { 130 this.name = name; 131 } 132 133 @Override 134 public int[] getShape() { 135 return shape.clone(); 136 } 137 138 @Override 139 public int getRank() { 140 return shape.length; 141 } 142 143 /** 144 * This method allows anything that dirties the dataset to clear various metadata values 145 * so that the other methods can work correctly. 146 * @since 2.1 147 */ 148 public void setDirty() { 149 dirty = true; 150 } 151 152 /** 153 * Find first sub-interface of (or class that directly implements) MetadataType 154 * @param clazz 155 * @return sub-interface 156 * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it 157 */ 158 @SuppressWarnings("unchecked") 159 public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) { 160 if (clazz.equals(MetadataType.class)) { 161 throw new IllegalArgumentException("Cannot accept MetadataType"); 162 } 163 164 if (clazz.isInterface()) { 165 return clazz; 166 } 167 168 if (clazz.isAnonymousClass()) { // special case 169 Class<?> s = clazz.getSuperclass(); 170 if (!s.equals(Object.class)) { 171 // only use super class if it is not an anonymous class of an interface 172 clazz = (Class<? extends MetadataType>) s; 173 } 174 } 175 176 for (Class<?> c : clazz.getInterfaces()) { 177 if (c.equals(MetadataType.class)) { 178 if (clazz.isAnonymousClass()) { 179 throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType"); 180 } 181 return clazz; 182 } 183 if (MetadataType.class.isAssignableFrom(c)) { 184 return (Class<? extends MetadataType>) c; 185 } 186 } 187 188 Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class 189 if (c != null) { 190 return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c); 191 } 192 193 logger.error("Somehow the search for metadata type interface ended in a bad place"); 194 assert false; // should not be able to get here!!! 195 return null; 196 } 197 198 @Override 199 public void setMetadata(MetadataType metadata) { 200 addMetadata(metadata, true); 201 } 202 203 @Override 204 public void addMetadata(MetadataType metadata) { 205 addMetadata(metadata, false); 206 } 207 208 private synchronized void addMetadata(MetadataType metadata, boolean clear) { 209 if (metadata == null) { 210 return; 211 } 212 213 if (this.metadata == null) { 214 this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 215 } 216 217 Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass()); 218 if (!this.metadata.containsKey(clazz)) { 219 this.metadata.put(clazz, new ArrayList<MetadataType>()); 220 } else if (clear) { 221 this.metadata.get(clazz).clear(); 222 } 223 this.metadata.get(clazz).add(metadata); 224 225 // add for special case of sub-interfaces of IMetadata 226 if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) { 227 clazz = IMetadata.class; 228 if (!this.metadata.containsKey(clazz)) { 229 this.metadata.put(clazz, new ArrayList<MetadataType>()); 230 } else if (clear) { 231 this.metadata.get(clazz).clear(); 232 } 233 this.metadata.get(clazz).add(metadata); 234 } 235 } 236 237 @Override 238 @Deprecated 239 public synchronized IMetadata getMetadata() { 240 return getFirstMetadata(IMetadata.class); 241 } 242 243 @SuppressWarnings("unchecked") 244 @Override 245 public synchronized <S extends MetadataType, T extends S> List<S> getMetadata(Class<T> clazz) throws MetadataException { 246 if (metadata == null) { 247 dirty = false; 248 return null; 249 } 250 251 if (dirty) { 252 dirtyMetadata(); 253 dirty = false; 254 } 255 256 if (clazz == null) { 257 List<S> all = new ArrayList<S>(); 258 for (Class<? extends MetadataType> c : metadata.keySet()) { 259 all.addAll((Collection<S>) metadata.get(c)); 260 } 261 return all; 262 } 263 264 return (List<S>) metadata.get(findMetadataTypeSubInterfaces(clazz)); 265 } 266 267 @Override 268 public synchronized <S extends MetadataType, T extends S> S getFirstMetadata(Class<T> clazz) { 269 try { 270 List<S> ml = getMetadata(clazz); 271 if (ml == null) { 272 return null; 273 } 274 for (S t : ml) { 275 if (clazz.isInstance(t)) { 276 return t; 277 } 278 } 279 } catch (Exception e) { 280 logger.error("Get metadata failed!",e); 281 } 282 283 return null; 284 } 285 286 @Override 287 public synchronized void clearMetadata(Class<? extends MetadataType> clazz) { 288 if (metadata == null) { 289 return; 290 } 291 292 if (clazz == null) { 293 metadata.clear(); 294 return; 295 } 296 297 List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz)); 298 if( list != null) { 299 list.clear(); 300 } 301 } 302 303 /** 304 * @since 2.0 305 */ 306 protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() { 307 return copyMetadata(metadata); 308 } 309 310 /** 311 * @since 2.0 312 */ 313 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) { 314 if (metadata == null) { 315 return null; 316 } 317 318 ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 319 copyMetadata(metadata, map); 320 return map; 321 } 322 323 private static void copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> inMetadata, 324 Map<Class<? extends MetadataType>, List<MetadataType>> outMetadata) { 325 for (Class<? extends MetadataType> c : inMetadata.keySet()) { 326 List<MetadataType> l = inMetadata.get(c); 327 List<MetadataType> nl = new ArrayList<MetadataType>(l.size()); 328 outMetadata.put(c, nl); 329 for (MetadataType m : l) { 330 if (m == null || isMetadataDirty(m)) { // skip dirty metadata 331 continue; 332 } 333 nl.add(m.clone()); 334 } 335 } 336 } 337 338 protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) { 339 copyMetadata(oldMetadata, metadata); 340 } 341 342 /** 343 * @since 2.2 344 */ 345 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> getMetadataMap(ILazyDataset a, boolean clone) { 346 List<MetadataType> all = null; 347 try { 348 all = a.getMetadata(null); 349 } catch (Exception e) { 350 } 351 if (all == null) { 352 return null; 353 } 354 355 ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 356 357 for (MetadataType m : all) { 358 if (m == null || isMetadataDirty(m)) { // skip dirty metadata 359 continue; 360 } 361 Class<? extends MetadataType> c = findMetadataTypeSubInterfaces(m.getClass()); 362 List<MetadataType> l = map.get(c); 363 if (l == null) { 364 l = new ArrayList<MetadataType>(); 365 map.put(c, l); 366 } 367 if (clone) { 368 m = m.clone(); 369 } 370 l.add(m); 371 } 372 return map; 373 } 374 375 private static boolean isMetadataDirty(MetadataType m) { 376 Class<? extends MetadataType> c = m.getClass(); 377 for (Field f : c.getDeclaredFields()) { 378 if (f.isAnnotationPresent(Dirtiable.class)) { 379 Class<?> t = f.getType(); 380 if (t.equals(boolean.class) || t.equals(Boolean.class)) { 381 try { 382 f.setAccessible(true); 383 Object o = f.get(m); 384 if (o.equals(true)) { 385 return true; 386 } 387 } catch (Exception e) { 388 logger.debug("Could not retrieve value of dirty variable: {}", c.getCanonicalName(), e); 389 } 390 } 391 } 392 } 393 394 return false; 395 } 396 397 interface MetadatasetAnnotationOperation { 398 /** 399 * Process value of given field 400 * <p> 401 * When the field is not a container then the returned value 402 * may replace the old value 403 * @param f given field 404 * @param o value of field 405 * @return transformed field 406 */ 407 Object processField(Field f, Object o); 408 409 /** 410 * @return annotated class 411 */ 412 Class<? extends Annotation> getAnnClass(); 413 414 /** 415 * @param axis 416 * @return number of dimensions to insert or remove 417 */ 418 int change(int axis); 419 420 /** 421 * 422 * @return rank or -1 to match 423 */ 424 int getNewRank(); 425 426 /** 427 * Run on given lazy dataset 428 * @param lz 429 * @return 430 */ 431 ILazyDataset run(ILazyDataset lz); 432 } 433 434 class MdsSlice implements MetadatasetAnnotationOperation { 435 private boolean asView; 436 private SliceND slice; 437 private int[] oShape; 438 private long oSize; 439 440 public MdsSlice(boolean asView, SliceND slice) { 441 this.asView = asView; 442 this.slice = slice; 443 oShape = slice.getSourceShape(); 444 oSize = ShapeUtils.calcLongSize(oShape); 445 } 446 447 @Override 448 public Object processField(Field field, Object o) { 449 return o; 450 } 451 452 @Override 453 public Class<? extends Annotation> getAnnClass() { 454 return Sliceable.class; 455 } 456 457 @Override 458 public int change(int axis) { 459 return 0; 460 } 461 462 @Override 463 public int getNewRank() { 464 return -1; 465 } 466 467 @Override 468 public ILazyDataset run(ILazyDataset lz) { 469 int rank = lz.getRank(); 470 if (slice.getStart().length != rank) { 471 throw new IllegalArgumentException("Slice rank does not match dataset!"); 472 } 473 474 int[] shape = lz.getShape(); 475 SliceND nslice; 476 if (!ShapeUtils.areShapesBroadcastCompatible(oShape, shape)) { 477 nslice = new SliceND(shape); 478 for (int i = 0; i < rank; i++) { 479 int s = shape[i]; 480 int os = oShape[i]; 481 if (s >= os) { 482 nslice.setSlice(i, 0, os, 1); 483 } else if (s == 1) { 484 nslice.setSlice(i, 0, 1, 1); 485 } else { 486 throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!"); 487 } 488 } 489 lz = lz.getSliceView(nslice); 490 } 491 if (lz.getSize() == oSize) { 492 nslice = slice; 493 } else { 494 nslice = slice.clone(); 495 for (int i = 0; i < rank; i++) { 496 int s = shape[i]; 497 if (s >= oShape[i]) { 498 continue; 499 } else if (s == 1) { 500 nslice.setSlice(i, 0, 1, 1); 501 } else { 502 throw new IllegalArgumentException("Sliceable dataset has non-unit dimension less than host!"); 503 } 504 } 505 } 506 507 if (asView || (lz instanceof IDataset)) { 508 return lz.getSliceView(nslice); 509 } 510 try { 511 return lz.getSlice(nslice); 512 } catch (DatasetException e) { 513 logger.error("Could not slice dataset in metadata", e); 514 return null; 515 } 516 } 517 } 518 519 class MdsReshape implements MetadatasetAnnotationOperation { 520 private boolean matchRank; 521 private int[] oldShape; 522 private int[] newShape; 523 boolean onesOnly; 524 int[] differences; 525 526 /* 527 * if only ones then record differences (insertions and deletions) 528 * 529 * if shape changing, find broadcasted dimensions and disallow 530 * merging that include those dimensions 531 */ 532 public MdsReshape(final int[] oldShape, final int[] newShape) { 533 this.oldShape = oldShape; 534 this.newShape = newShape; 535 differences = null; 536 } 537 538 @Override 539 public Object processField(Field field, Object o) { 540 Annotation a = field.getAnnotation(Reshapeable.class); 541 if (a != null) { // cannot be null 542 matchRank = ((Reshapeable) a).matchRank(); 543 } 544 return o; 545 } 546 547 @Override 548 public Class<? extends Annotation> getAnnClass() { 549 return Reshapeable.class; 550 } 551 552 @Override 553 public int change(int axis) { 554 if (matchRank) { 555 if (differences == null) { 556 init(); 557 } 558 559 if (onesOnly) { 560 return differences == null ? 0 : differences[axis]; 561 } 562 throw new UnsupportedOperationException("TODO support other shape operations"); 563 } 564 return 0; 565 } 566 567 @Override 568 public int getNewRank() { 569 return matchRank ? newShape.length : -1; 570 } 571 572 private void init() { 573 int or = oldShape.length - 1; 574 int nr = newShape.length - 1; 575 if (or < 0 || nr < 0) { // zero-rank shapes 576 onesOnly = true; 577 differences = new int[1]; 578 differences[0] = or < 0 ? nr + 1 : or + 1; 579 return; 580 } 581 onesOnly = ShapeUtils.differsByOnes(oldShape, newShape); 582 int ob = 0; 583 int nb = 0; 584 if (onesOnly) { 585 differences = ShapeUtils.calcShapePadding(oldShape, newShape); 586 } else { 587 differences = new int[or + 2]; 588 if (matchRank) { 589 logger.error("Combining dimensions is currently not supported"); 590 throw new IllegalArgumentException("Combining dimensions is currently not supported"); 591 } 592 // work out mapping: contiguous dimensions can be grouped or split 593 while (ob <= or && nb <= nr) { 594 int ol = oldShape[ob]; 595 while (ol == 1 && ol <= or) { 596 ob++; 597 ol = oldShape[ob]; 598 } 599 int oe = ob + 1; 600 int nl = newShape[nb]; 601 while (nl == 1 && nl <= nr) { 602 nb++; 603 nl = newShape[nb]; 604 } 605 int ne = nb + 1; 606 if (ol < nl) { 607 differences[ob] = 1; 608 do { // case where new shape combines several dimensions into one dimension 609 if (oe == (or + 1)) { 610 break; 611 } 612 differences[oe] = 1; 613 ol *= oldShape[oe++]; 614 } while (ol < nl); 615 differences[oe - 1] = oe - ob; // signal end with difference 616 if (nl != ol) { 617 logger.error("Single dimension is incompatible with subshape"); 618 throw new IllegalArgumentException("Single dimension is incompatible with subshape"); 619 } 620 } else if (ol > nl) { 621 do { // case where new shape spreads single dimension over several dimensions 622 if (ne == (nr + 1)) { 623 break; 624 } 625 nl *= newShape[ne++]; 626 } while (nl < ol); 627 if (nl != ol) { 628 logger.error("Subshape is incompatible with single dimension"); 629 throw new IllegalArgumentException("Subshape is incompatible with single dimension"); 630 } 631 } 632 633 ob = oe; 634 nb = ne; 635 } 636 } 637 } 638 639 @Override 640 public ILazyDataset run(ILazyDataset lz) { 641 if (differences == null) { 642 init(); 643 } 644 645 int[] lshape = lz.getShape(); 646 if (Arrays.equals(newShape, lshape)) { 647 return lz; 648 } 649 int or = lshape.length; 650 int nr = newShape.length; 651 int[] nshape; 652 if (onesOnly) { 653 nshape = ShapeUtils.padShape(differences, nr, lshape); 654 } else { 655 nshape = new int[nr]; 656 boolean[] broadcast = new boolean[or]; 657 for (int ob = 0; ob < or; ob++) { 658 broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1; 659 } 660 int osize = lz.getSize(); 661 662 // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...) 663 int ob = 0; 664 int nsize = 1; 665 for (int i = 0; i < nr; i++) { 666 if (ob < or && broadcast[ob]) { 667 if (differences[ob] != 0) { 668 logger.error("Metadata contains a broadcast axis which cannot be reshaped"); 669 throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped"); 670 } 671 nshape[i] = 1; 672 } else { 673 nshape[i] = nsize < osize ? newShape[i] : 1; 674 } 675 nsize *= nshape[i]; 676 ob++; 677 } 678 } 679 680 ILazyDataset nlz; 681 if (lz instanceof Dataset) { 682 nlz = ((Dataset) lz).reshape(nshape); 683 } else { 684 nlz = lz.getSliceView(); 685 nlz.setShape(nshape); 686 } 687 return nlz; 688 } 689 } 690 691 class MdsTranspose implements MetadatasetAnnotationOperation { 692 int[] map; 693 694 public MdsTranspose(final int[] axesMap) { 695 map = axesMap; 696 } 697 698 @SuppressWarnings({ "rawtypes", "unchecked" }) 699 @Override 700 public Object processField(Field f, Object o) { 701 // reorder arrays and lists according the axes map 702 if (o.getClass().isArray()) { 703 int l = Array.getLength(o); 704 if (l == map.length) { 705 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 706 for (int i = 0; i < l; i++) { 707 Array.set(narray, i, Array.get(o, map[i])); 708 } 709 for (int i = 0; i < l; i++) { 710 Array.set(o, i, Array.get(narray, i)); 711 } 712 } 713 } else if (o instanceof List<?>) { 714 List list = (List) o; 715 int l = list.size(); 716 if (l == map.length) { 717 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 718 for (int i = 0; i < l; i++) { 719 Array.set(narray, i, list.get(map[i])); 720 } 721 list.clear(); 722 for (int i = 0; i < l; i++) { 723 list.add(Array.get(narray, i)); 724 } 725 } 726 } 727 return o; 728 } 729 730 @Override 731 public Class<? extends Annotation> getAnnClass() { 732 return Transposable.class; 733 } 734 735 @Override 736 public int change(int axis) { 737 return 0; 738 } 739 740 @Override 741 public int getNewRank() { 742 return -1; 743 } 744 745 @Override 746 public ILazyDataset run(ILazyDataset lz) { 747 return lz.getTransposedView(map); 748 } 749 } 750 751 class MdsDirty implements MetadatasetAnnotationOperation { 752 753 @Override 754 public Object processField(Field f, Object o) { 755 // throw exception if not boolean??? 756 Class<?> t = f.getType(); 757 if (t.equals(boolean.class) || t.equals(Boolean.class)) { 758 if (o.equals(false)) { 759 o = true; 760 } 761 } 762 return o; 763 } 764 765 @Override 766 public Class<? extends Annotation> getAnnClass() { 767 return Dirtiable.class; 768 } 769 770 @Override 771 public int change(int axis) { 772 return 0; 773 } 774 775 @Override 776 public int getNewRank() { 777 return -1; 778 } 779 780 @Override 781 public ILazyDataset run(ILazyDataset lz) { 782 return lz; 783 } 784 } 785 786 /** 787 * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced 788 * dataset after cloning the metadata 789 * @param asView if true then just a view 790 * @param slice 791 */ 792 protected void sliceMetadata(boolean asView, final SliceND slice) { 793 processAnnotatedMetadata(new MdsSlice(asView, slice), true); 794 } 795 796 /** 797 * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing 798 * or setting the shape 799 * 800 * @param newShape 801 */ 802 protected void reshapeMetadata(final int[] oldShape, final int[] newShape) { 803 processAnnotatedMetadata(new MdsReshape(oldShape, newShape), true); 804 } 805 806 /** 807 * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed 808 * dataset after cloning the metadata 809 * @param axesMap 810 */ 811 protected void transposeMetadata(final int[] axesMap) { 812 processAnnotatedMetadata(new MdsTranspose(axesMap), true); 813 } 814 815 /** 816 * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified 817 * @since 2.0 818 */ 819 protected void dirtyMetadata() { 820 processAnnotatedMetadata(new MdsDirty(), true); 821 } 822 823 @SuppressWarnings("unchecked") 824 private void processAnnotatedMetadata(MetadatasetAnnotationOperation op, boolean throwException) { 825 if (metadata == null) 826 return; 827 828 for (List<MetadataType> l : metadata.values()) { 829 for (MetadataType m : l) { 830 if (m == null) { 831 continue; 832 } 833 834 Class<? extends MetadataType> mc = m.getClass(); 835 do { // iterate over super-classes 836 processClass(op, m, mc, throwException); 837 Class<?> sclazz = mc.getSuperclass(); 838 if (!MetadataType.class.isAssignableFrom(sclazz)) { 839 break; 840 } 841 mc = (Class<? extends MetadataType>) sclazz; 842 } while (true); 843 } 844 } 845 } 846 847 @SuppressWarnings({ "unchecked", "rawtypes" }) 848 private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc, boolean throwException) { 849 for (Field f : mc.getDeclaredFields()) { 850 if (!f.isAnnotationPresent(op.getAnnClass())) 851 continue; 852 853 try { 854 f.setAccessible(true); 855 Object o = f.get(m); 856 if (o == null) { 857 continue; 858 } 859 860 Object no = op.processField(f, o); 861 if (no != o) { 862 f.set(m, no); 863 continue; 864 } 865 Object r = null; 866 if (o instanceof ILazyDataset) { 867 try { 868 f.set(m, op.run((ILazyDataset) o)); 869 } catch (Exception e) { 870 logger.error("Problem processing " + o, e); 871 if (!catchExceptions) { 872 throw e; 873 } 874 } 875 } else if (o.getClass().isArray()) { 876 int l = Array.getLength(o); 877 878 for (int i = 0; r == null && i < l; i++) { 879 r = Array.get(o, i); 880 } 881 int n = op.getNewRank(); 882 if (r == null) { 883 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 884 f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n)); 885 } 886 continue; 887 } 888 if (n < 0) { 889 n = l; 890 } 891 Object narray = Array.newInstance(r.getClass(), n); 892 for (int i = 0, si = 0, di = 0; di < n && si < l; i++) { 893 int c = op.change(i); 894 if (c == 0) { 895 Array.set(narray, di++, processObject(op, Array.get(o, si++))); 896 } else if (c > 0) { 897 di += c; // add nulls by skipping forward in destination array 898 } else if (c < 0) { 899 si -= c; // remove dimensions by skipping forward in source array 900 } 901 } 902 if (n == l) { 903 for (int i = 0; i < l; i++) { 904 Array.set(o, i, Array.get(narray, i)); 905 } 906 } else { 907 f.set(m, narray); 908 } 909 } else if (o instanceof List<?>) { 910 List list = (List) o; 911 int l = list.size(); 912 913 for (int i = 0; r == null && i < l; i++) { 914 r = list.get(i); 915 } 916 int n = op.getNewRank(); 917 if (r == null) { 918 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 919 list.clear(); 920 for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) { 921 list.add(null); 922 } 923 } 924 continue; 925 } 926 927 if (n < 0) { 928 n = l; 929 } 930 Object narray = Array.newInstance(r.getClass(), n); 931 for (int i = 0, si = 0, di = 0; i < l && si < l; i++) { 932 int c = op.change(i); 933 if (c == 0) { 934 Array.set(narray, di++, processObject(op, list.get(si++))); 935 } else if (c > 0) { 936 di += c; // add nulls by skipping forward in destination array 937 } else if (c < 0) { 938 si -= c; // remove dimensions by skipping forward in source array 939 } 940 } 941 list.clear(); 942 for (int i = 0; i < n; i++) { 943 list.add(Array.get(narray, i)); 944 } 945 } else if (o instanceof Map<?,?>) { 946 Map map = (Map) o; 947 for (Object k : map.keySet()) { 948 map.put(k, processObject(op, map.get(k))); 949 } 950 } 951 } catch (Exception e) { 952 logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e); 953 if (throwException) { 954 throw new RuntimeException(e); 955 } 956 } 957 } 958 } 959 960 @SuppressWarnings({ "unchecked", "rawtypes" }) 961 private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception { 962 if (o == null) { 963 return o; 964 } 965 966 if (o instanceof ILazyDataset) { 967 try { 968 return op.run((ILazyDataset) o); 969 } catch (Exception e) { 970 logger.error("Problem processing " + o, e); 971 if (!catchExceptions) { 972 throw e; 973 } 974 } 975 } else if (o.getClass().isArray()) { 976 int l = Array.getLength(o); 977 for (int i = 0; i < l; i++) { 978 Array.set(o, i, processObject(op, Array.get(o, i))); 979 } 980 } else if (o instanceof List<?>) { 981 List list = (List) o; 982 for (int i = 0, imax = list.size(); i < imax; i++) { 983 list.set(i, processObject(op, list.get(i))); 984 } 985 } else if (o instanceof Map<?,?>) { 986 Map map = (Map) o; 987 for (Object k : map.keySet()) { 988 map.put(k, processObject(op, map.get(k))); 989 } 990 } 991 return o; 992 } 993 994 protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) { 995 ILazyDataset d = null; 996 if (blob instanceof ILazyDataset) { 997 d = (ILazyDataset) blob; 998 if (d instanceof IDataset) { 999 Dataset ed = DatasetUtils.convertToDataset((IDataset) d); 1000 int is = ed.getElementsPerItem(); 1001 if (is != 1 && is != getElementsPerItem()) { 1002 throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset"); 1003 } 1004 d = ed.cast(is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64); 1005 } else if (!keepLazy) { 1006 final int is = getElementsPerItem(); 1007 try { 1008 d = DatasetUtils.cast(d.getSlice(), is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64); 1009 } catch (DatasetException e) { 1010 logger.error("Could not get data from lazy dataset", e); 1011 return null; 1012 } 1013 } 1014 } else { 1015 final int is = getElementsPerItem(); 1016 if (is == 1) { 1017 d = DatasetFactory.createFromObject(DoubleDataset.class, blob); 1018 } else { 1019 try { 1020 d = DatasetFactory.createFromObject(is, CompoundDoubleDataset.class, blob); 1021 } catch (IllegalArgumentException e) { // if only single value supplied try again 1022 d = DatasetFactory.createFromObject(DoubleDataset.class, blob); 1023 } 1024 } 1025 if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) { 1026 d.setShape(shape.clone()); 1027 } 1028 } 1029 List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape()); 1030 d.setShape(s.get(0)); 1031 1032 return d; 1033 } 1034 1035 @Override 1036 public void setErrors(Serializable errors) { 1037 if (shape == null) { 1038 throw new IllegalArgumentException("Cannot set errors for null dataset"); 1039 } 1040 if (errors == null) { 1041 clearMetadata(ErrorMetadata.class); 1042 return; 1043 } 1044 if (errors == this) { 1045 logger.warn("Ignoring setting error to itself as this will lead to infinite recursion"); 1046 return; 1047 } 1048 1049 ILazyDataset errorData = createFromSerializable(errors, true); 1050 1051 ErrorMetadata emd = getErrorMetadata(); 1052 if (emd == null) { 1053 try { 1054 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1055 setMetadata(emd); 1056 } catch (MetadataException me) { 1057 logger.error("Could not create metadata", me); 1058 } 1059 } 1060 emd.setError(errorData); 1061 } 1062 1063 protected ErrorMetadata getErrorMetadata() { 1064 try { 1065 List<ErrorMetadata> el = getMetadata(ErrorMetadata.class); 1066 if (el != null && !el.isEmpty()) { 1067 return el.get(0); 1068 } 1069 } catch (Exception e) { 1070 } 1071 return null; 1072 } 1073 1074 @Override 1075 public ILazyDataset getErrors() { 1076 ErrorMetadata emd = getErrorMetadata(); 1077 return emd == null ? null : emd.getError(); 1078 } 1079 1080 @Override 1081 public boolean hasErrors() { 1082 return LazyDatasetBase.this.getErrors() != null; 1083 } 1084 1085 /** 1086 * Check permutation axes 1087 * @param shape 1088 * @param axes 1089 * @return cleaned up copy of axes or null if trivial 1090 */ 1091 public static int[] checkPermutatedAxes(int[] shape, int... axes) { 1092 int rank = shape == null ? 0 : shape.length; 1093 1094 if (axes == null || axes.length == 0) { 1095 axes = new int[rank]; 1096 for (int i = 0; i < rank; i++) { 1097 axes[i] = rank - 1 - i; 1098 } 1099 } else { 1100 axes = axes.clone(); 1101 } 1102 1103 if (axes.length != rank) { 1104 logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank); 1105 throw new IllegalArgumentException("axis permutation does not match shape of dataset"); 1106 } 1107 1108 // check all permutation values are within bounds 1109 for (int i = 0; i < rank; i++) { 1110 axes[i] = ShapeUtils.checkAxis(rank, axes[i]); 1111 } 1112 1113 // check for a valid permutation (is this an unnecessary restriction?) 1114 int[] perm = axes.clone(); 1115 Arrays.sort(perm); 1116 1117 for (int i = 0; i < rank; i++) { 1118 if (perm[i] != i) { 1119 logger.error("axis permutation is not valid: it does not contain complete set of axes"); 1120 throw new IllegalArgumentException("axis permutation does not contain complete set of axes"); 1121 } 1122 } 1123 1124 if (Arrays.equals(axes, perm)) { 1125 return null; // signal identity or trivial permutation 1126 } 1127 1128 return axes; 1129 } 1130}