Fixed warnings in a lot of places.

Made MPI_Comm_split happen on-demand rather than at start-up for all the torus topologies.

Added MPI_Finalized(..) check to make sure that CTF exits gracefully (without freeing the main comm)
if a user calls MPI_Finalize() before destroying a CTF_World

Makefile

@@ -38,11 +38,12 @@ studies: $(STUDIES)
 $(STUDIES):
 	$(MAKE) $@ -C src/studies
 
-lib/libctf.a: ctf
+lib/libctf.a: objs 
 	$(AR) -crs $@ src/*/*.o
 
-ctf: src 
-	$(MAKE) $@ -C src
+.PHONY: objs
+objs: src/*/*
+	$(MAKE) ctf -C src
 	
 clean: clean_bin clean_lib
 	$(MAKE) $@ -C src

examples/Makefile

@@ -7,7 +7,7 @@ EXAMPLES = dft dft_3D gemm gemm_4D scalar trace weight_4D subworld_gemm \
 .PHONY: $(EXAMPLES)
 $(EXAMPLES): %:  ../bin/%
 
-../bin/%: %.cxx 
+../bin/%: %.cxx ../lib/libctf.a
 	$(FCXX) $< -o $@ -I../include/ -L../lib -lctf $(LIBS)
 
 

examples/ccsd.cxx

@@ -184,8 +184,8 @@ void ccsd(Integrals   &V,
           Amplitudes  &T,
           int sched_nparts = 0){
   int rank;   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-  double timer = MPI_Wtime();
 #ifdef SCHEDULE_CCSD
+  double timer = MPI_Wtime();
   tCTF_Schedule<double> sched(V.dw);
   sched.set_max_partitions(sched_nparts);
   sched.record();

examples/dft.cxx

@@ -84,7 +84,7 @@ int test_dft(int64_t const  n,
  * \brief Forms N-by-N DFT matrix A and inverse-dft iA and checks A*iA=I
  */
 int main(int argc, char ** argv){
-  int myRank, numPes, logn;
+  int logn;
   int64_t n;
   
   MPI_Init(&argc, &argv);

examples/gemm.cxx

@@ -141,7 +141,7 @@ char* getCmdOption(char ** begin,
 }
 
 int main(int argc, char ** argv){
-  int rank, np, niter, n, m, k, pass;
+  int rank, np, niter, n, m, k;
   int const in_num = argc;
   char ** input_str = argv;
 

examples/permute_multiworld.cxx

@@ -177,7 +177,7 @@ char* getCmdOption(char ** begin,
 }
 
 int main(int argc, char ** argv){
-  int rank, np, niter, n, pass;
+  int rank, np, n;
   int const in_num = argc;
   char ** input_str = argv;
 

examples/scalar.cxx

@@ -122,9 +122,6 @@ char* getCmdOption(char ** begin,
 
 int main(int argc, char ** argv){
   int rank, np;
-  int const in_num = argc;
-  char ** input_str = argv;
-
   MPI_Init(&argc, &argv);
   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
   MPI_Comm_size(MPI_COMM_WORLD, &np);

examples/slice_gemm.cxx

@@ -142,7 +142,7 @@ char* getCmdOption(char ** begin,
 }
 
 int main(int argc, char ** argv){
-  int rank, np, niter, n, m, k, pass;
+  int rank, np, n, m, k;
   int const in_num = argc;
   char ** input_str = argv;
 

examples/sparse_permuted_slice.cxx

@@ -161,7 +161,7 @@ char* getCmdOption(char ** begin,
 }
 
 int main(int argc, char ** argv){
-  int rank, np, niter, n, b, pass;
+  int rank, np, n, b;
   int const in_num = argc;
   char ** input_str = argv;
 

examples/strassen.cxx

@@ -262,7 +262,7 @@ char* getCmdOption(char ** begin,
 }
 
 int main(int argc, char ** argv){
-  int rank, np, n,  pass;
+  int rank, np, n;
   int const in_num = argc;
   char ** input_str = argv;
 

examples/subworld_gemm.cxx

@@ -116,7 +116,7 @@ char* getCmdOption(char ** begin,
 }
 
 int main(int argc, char ** argv){
-  int rank, np, niter, n, m, k, pass, div;
+  int rank, np, n, m, k, div;
   int const in_num = argc;
   char ** input_str = argv;
 

include/ctf.hpp

@@ -1254,10 +1254,10 @@ public:
 	tCTF_TensorOperation(tCTF_TensorOperationTypes op,
 			tCTF_Idx_Tensor<dtype>* lhs,
 			const tCTF_Term<dtype>* rhs) :
+			  dependency_count(0),
 			  op(op),
 			  lhs(lhs),
 			  rhs(rhs),
-			  dependency_count(0),
 			  cached_estimated_cost(0) {}
 
   /**

src/bench/Makefile

@@ -15,5 +15,5 @@ else
 endif
 
 
-../../bin/%: %.cxx 
+../../bin/%: %.cxx ../../lib/libctf.a
 	$(FCXX) $< -o $@ -I../../include/ -L../../lib -lctf $(LIBS)

src/bench/bench_contraction.cxx

@@ -86,7 +86,7 @@ char* getCmdOption(char ** begin,
 
 
 int main(int argc, char ** argv){
-  int rank, np, niter, n, m;
+  int rank, np, niter, n;
   int const in_num = argc;
   char ** input_str = argv;
   char const * A;

src/bench/nonsq_pgemm_bench.cxx

@@ -128,7 +128,6 @@ int main(int argc, char **argv) {
   int myRank, numPes;
   int m, k, n;
   int nprow, npcol;
-  int tid_A, tid_B, tid_C;
 
   MPI_Init(&argc, &argv);
   MPI_Comm_size(MPI_COMM_WORLD, &numPes);

src/ctr_comm/Makefile

 include ../../config.mk
 
-OBJS = ctr_2d_general.o ctr_simple.o ctr_tsr.o strp_tsr.o sum_tsr.o scale_tsr.o strp_tsr.o ctr_offload.o
+OBJS = ctr_2d_general.o ctr_simple.o ctr_tsr.o sum_tsr.o scale_tsr.o strp_tsr.o ctr_offload.o
 
 ctf: $(OBJS) 
 

src/ctr_comm/ctr_2d_general.cxx

@@ -9,6 +9,9 @@
  */
 template<typename dtype>
 ctr_2d_general<dtype>::~ctr_2d_general() {
+  if (move_A) FREE_CDT(cdt_A);
+  if (move_B) FREE_CDT(cdt_B);
+  if (move_C) FREE_CDT(cdt_C);
   if (rec_ctr != NULL)
     delete rec_ctr;
 }

src/ctr_comm/ctr_comm.h

@@ -104,7 +104,7 @@ class ctr_2d_general : public ctr<dtype> {
 
     ctr_2d_general(ctr<dtype> * other);
     ~ctr_2d_general();
-    ctr_2d_general(){}
+    ctr_2d_general(){ move_A=0; move_B=0; move_C=0; }
 };
 
 template<typename dtype>

src/ctr_comm/ctr_simple.cxx

@@ -211,10 +211,19 @@ void ctr_lyr<dtype>::run(){
 template<typename dtype>
 ctr_replicate<dtype>::~ctr_replicate() {
   delete rec_ctr;
+  for (int i=0; i<ncdt_A; i++){
+    FREE_CDT(cdt_A[i]);
+  }
   if (ncdt_A > 0)
     CTF_free(cdt_A);
+  for (int i=0; i<ncdt_B; i++){
+    FREE_CDT(cdt_B[i]);
+  }
   if (ncdt_B > 0)
     CTF_free(cdt_B);
+  for (int i=0; i<ncdt_C; i++){
+    FREE_CDT(cdt_C[i]);
+  }
   if (ncdt_C > 0)
     CTF_free(cdt_C);
 }

src/ctr_comm/sum_tsr.cxx

@@ -147,8 +147,14 @@ do {                                                                    \
 template<typename dtype>
 tsum_replicate<dtype>::~tsum_replicate() {
   delete rec_tsum;
+  for (int i=0; i<ncdt_A; i++){
+    FREE_CDT(cdt_A[i]);
+  }
   if (ncdt_A > 0)
     CTF_free(cdt_A);
+  for (int i=0; i<ncdt_B; i++){
+    FREE_CDT(cdt_B[i]);
+  }
   if (ncdt_B > 0)
     CTF_free(cdt_B);
 }