tiledarray.cpp

Go to the documentation of this file.

#include <TiledArray/config.h>
#include <TiledArray/initialize.h>
 
#ifdef TILEDARRAY_HAS_CUDA
#include <TiledArray/cuda/cublas.h>
#include <TiledArray/external/cuda.h>
#include <cutt.h>
#endif
 
#ifdef TILEDARRAY_HAS_INTEL_MKL
#include <mkl_service.h>
#endif
 
namespace TiledArray {
namespace {
 
#ifdef TILEDARRAY_HAS_CUDA
inline void cuda_initialize() {
  cudaEnv::instance();
  //
  cuBLASHandlePool::handle();
  // initialize cuTT
  cuttInitialize();
}
 
inline void cuda_finalize() {
  CudaSafeCall(cudaDeviceSynchronize());
  cuttFinalize();
  cublasDestroy(cuBLASHandlePool::handle());
  delete &cuBLASHandlePool::handle();
  cudaEnv::instance().reset(nullptr);
}
#endif
 
inline bool& initialized_madworld_accessor() {
  static bool flag = false;
  return flag;
}
inline bool initialized_madworld() { return initialized_madworld_accessor(); }
inline bool& initialized_accessor() {
  static bool flag = false;
  return flag;
}
inline bool& finalized_accessor() {
  static bool flag = false;
  return flag;
}
#ifdef TILEDARRAY_HAS_INTEL_MKL
inline int& mklnumthreads_accessor() {
  static int value = -1;
  return value;
}
#endif
 
}  // namespace
}  // namespace TiledArray
 
bool TiledArray::initialized() { return initialized_accessor(); }
 
bool TiledArray::finalized() { return finalized_accessor(); }
 
 
 
TiledArray::World& TiledArray::initialize(int& argc, char**& argv,
                                          const SafeMPI::Intracomm& comm,
                                          bool quiet) {
  if (initialized_madworld() && finalized())
    throw Exception(
        "TiledArray finalized MADWorld already, cannot re-initialize MADWorld "
        "again");
  if (!initialized()) {
    if (!madness::initialized())
      initialized_madworld_accessor() = true;
    else {  // if MADWorld initialized, we must assume that comm is its default
            // World.
      if (madness::World::is_default(comm))
        throw Exception(
            "MADWorld initialized before TiledArray::initialize(argc, argv, "
            "comm), but not initialized with comm");
    }
    auto& default_world = initialized_madworld()
                              ? madness::initialize(argc, argv, comm, quiet)
                              : *madness::World::find_instance(comm);
    TiledArray::set_default_world(default_world);
#ifdef TILEDARRAY_HAS_CUDA
    TiledArray::cuda_initialize();
#endif
#ifdef TILEDARRAY_HAS_INTEL_MKL
    // record number of MKL threads and set to 1
    mklnumthreads_accessor() = mkl_get_max_threads();
    mkl_set_num_threads(1);
#endif
    madness::print_meminfo_disable();
    initialized_accessor() = true;
    return default_world;
  } else
    throw Exception("TiledArray already initialized");
}
 
void TiledArray::finalize() {
#ifdef TILEDARRAY_HAS_INTEL_MKL
  // reset number of MKL threads
  mkl_set_num_threads(mklnumthreads_accessor());
#endif
#ifdef TILEDARRAY_HAS_CUDA
  TiledArray::cuda_finalize();
#endif
  TiledArray::get_default_world()
      .gop.fence();  // TODO remove when madness::finalize() fences
  if (initialized_madworld()) {
    madness::finalize();
  }
  TiledArray::reset_default_world();
  initialized_accessor() = false;
  finalized_accessor() = true;
}
 
void TiledArray::ta_abort() {
  std::abort();
}
 
void TiledArray::ta_abort(const std::string &m) {
  std::cerr << m << std::endl;
  ta_abort();
}