nvcc under linux complains: Contains a vector, which is not supported in device code - linux

Question

I have the following code
#include <cuda.h>
#include <cuda_runtime.h>
#ifdef _MSC_VER
#include <intrin.h>
#else
#include <x86intrin.h>
#endif
//A bitset for the variable assignments
//The state for non existing variable 0 is stored as well, just to avoid +1/-1 adjustments
struct Atom_t {
enum where { device, host};
enum BoolOp {opXor, opOr, opAnd };
public: //TODO make private later
int VarCount;
bool isValid;
union {
uint32_t raw[1]; //don't worry about alignment, the compiler will not use aligned read/writes anyway.}
uint64_t raw64[1];
__m256i avxraw[1];
};
public:
__host__ __device__ friend bool operator==(const Atom_t& a, const Atom_t& b);
};
__host__ __device__ bool operator==(const Atom_t& a, const Atom_t& b) {
const auto IntCount = a.IntCount();
if (IntCount != b.IntCount()) { return false; }
#ifdef __CUDA_ARCH__
__shared__ bool isDifferent;
isDifferent = false;
for (auto i = ThreadId(); i < IntCount; i += BlockDim()) {
if (a.raw[i] != b.raw[i] || isDifferent) {
isDifferent = true;
break;
}
}
syncthreads();
return !isDifferent;
#else
auto result = true;
#ifdef _DEBUG
for (auto i = 0; i < IntCount; i++) {
if (a.raw[i] != b.raw[i]) { result = false; }
}
#endif
auto AvxCount = a.Avx2Count();
if (AvxCount != b.Avx2Count()) { if (result) { print("Atom_t == is incorrect"); } assert1(!result); return false; }
for (auto i = 0; i < AvxCount; i++) {
const auto packedCompare = _mm256_cmpeq_epi8(a.avxraw[i], b.avxraw[i]);
const auto bitmask = _mm256_movemask_epi8(packedCompare);
if (bitmask != -1) { if (result) { print("Atom_t == is incorrect"); } assert1(!result); return false; }
}
#endif
#ifndef __CUDA_ARCH__
assert(result);
#endif
return true;
}
The compiler complains
Description Resource Path Location Type
"__nv_bool (const Atom_t &, const Atom_t &)" contains a vector, which is not supported in device code
However, the vector is not in device code, only in the host code. How do I make this error go away in NSight Eclipse Edition 9.1 running CUDA 11.
I tried:
#ifdef __CUDA_ARCH__
# define DEAL_II_COMPILER_VECTORIZATION_LEVEL 0
#endif
But that does not work.

Answer 1

However, the vector is not in device code, only in the host code.
The error is coming about due to this line:
__m256i avxraw[1];
which is visible in both the host code and device code compilation trajectory.
According to my testing this may be a possible workaround:
$ cat t32.cpp
#ifdef _MSC_VER
#include <intrin.h>
#else
#include <x86intrin.h>
#endif
#include <iostream>
typedef char dummy[sizeof(__m256i)];
struct Atom_t {
enum where { device, host};
enum BoolOp {opXor, opOr, opAnd };
public: //TODO make private later
int VarCount;
bool isValid;
union {
uint32_t raw[1];
uint64_t raw64[1];
#ifndef FOO //hide the vectorized datastruct from cuda's view
__m256i avxraw[1];
#else
alignas(32) dummy foo[1];
#endif
};
};
int main(){
std::cout << sizeof(__m256i) << std::endl;
std::cout << sizeof(Atom_t) << std::endl;
}
$ g++ t32.cpp -o t32
$ ./t32
32
64
$ g++ t32.cpp -o t32 -DFOO
$ ./t32
32
64
(Fedora Core 29)
The alignas(32) directive is still probably somewhat fragile if the definition of __m256i changes dramatically. And, clearly, the above is not CUDA code in the exact frame that was presented. It would need to be adapted (e.g. replace #ifndef FOO with #ifndef __CUDA_ARCH__)
I'm not suggesting that this code is correct, defect-free, or suitable for any particular purpose; it is mostly code provided by OP. My objective here is to identify issues that I see and are asked about in the question, and suggest possible ways to address those issues. Use this at your own risk.

Answer 2

Found it!
The problem is not the code in the method, the problem is the presence of the _m256i within view of cuda.
The following patch fixes the issue:
struct Atom_t {
enum where { device, host};
enum BoolOp {opXor, opOr, opAnd };
public: //TODO make private later
int VarCount;
bool isValid;
union {
uint32_t raw[1]; //don't worry about alignment, the compiler will not use aligned read/writes anyway.}
uint64_t raw64[1];
#ifndef __CUDA_ARCH__ //hide the vectorized datastruct from cuda's view
__m256i avxraw[1];
#endif
};
Now that nvcc does not see the vectorized datatype it will stop worrying.