|
| 1 | +#include "ssm-conv.cuh" |
| 2 | + |
| 3 | +template <size_t split_d_inner, size_t d_conv> |
| 4 | +static __global__ void ssm_conv_f32(const float * __restrict__ src0, const float * __restrict__ src1, |
| 5 | + const int src0_nb0, const int src0_nb1, const int src0_nb2, const int src1_nb1, |
| 6 | + float * __restrict__ dst, const int dst_nb0, const int dst_nb1, const int dst_nb2, |
| 7 | + const int nc, const int ncs, const int nr, const int n_t, const int n_s) { |
| 8 | + const int tid = threadIdx.x; |
| 9 | + const int bidx = blockIdx.x; |
| 10 | + const int bidy = blockIdx.y; |
| 11 | + |
| 12 | + const float * x_block = (const float *) ((char *) src0 + bidx * src0_nb2 + bidy * split_d_inner * src0_nb1); |
| 13 | + const float * w_block = (const float *) ((char *) src1 + bidy * split_d_inner * src1_nb1); |
| 14 | + float * y_block = (float *) ((char *) dst + bidx * dst_nb2 + bidy * split_d_inner * dst_nb0); |
| 15 | + |
| 16 | + const int stride_x = src0_nb1 / sizeof(float); |
| 17 | + const int stride_w = src1_nb1 / sizeof(float); |
| 18 | + const int stride_y = dst_nb1 / sizeof(float); |
| 19 | + |
| 20 | + float x[d_conv] = { 0.0f }; |
| 21 | + float w[d_conv] = { 0.0f }; |
| 22 | + |
| 23 | +#pragma unroll |
| 24 | + for (int j = 0; j < d_conv; j++) { |
| 25 | + w[j] = w_block[tid * stride_w + j]; |
| 26 | + } |
| 27 | + |
| 28 | + for (int i = 0; i < n_t; i++) { |
| 29 | + float sumf = 0.0f; |
| 30 | + |
| 31 | + if (i == 0) { |
| 32 | + for (int j = 0; j < d_conv; j++) { |
| 33 | + x[j] = x_block[tid * stride_x + j]; |
| 34 | + } |
| 35 | + } else { |
| 36 | + x[(i - 1) % d_conv] = x_block[tid * stride_x + i + d_conv - 1]; |
| 37 | + } |
| 38 | + |
| 39 | +#pragma unroll |
| 40 | + for (int j = 0; j < d_conv; j++) { |
| 41 | + sumf += x[(i + j) % d_conv] * w[j]; |
| 42 | + } |
| 43 | + y_block[i * stride_y + tid] = sumf; |
| 44 | + } |
| 45 | +} |
| 46 | + |
| 47 | +template <size_t split_d_inner, size_t d_conv, size_t split_n_t> |
| 48 | +static __global__ void ssm_conv_long_token_f32(const float * __restrict__ src0, const float * __restrict__ src1, |
| 49 | + const int src0_nb0, const int src0_nb1, const int src0_nb2, |
| 50 | + const int src1_nb1, float * __restrict__ dst, const int dst_nb0, |
| 51 | + const int dst_nb1, const int dst_nb2, const int nc, const int ncs, |
| 52 | + const int nr, const int n_t, const int n_s) { |
| 53 | + const int tid = threadIdx.x; |
| 54 | + const int bidx = blockIdx.x; |
| 55 | + const int bidy = blockIdx.y; |
| 56 | + const int bidz = blockIdx.z; |
| 57 | + |
| 58 | + const float * x_block = (const float *) ((char *) src0 + bidx * src0_nb2 + bidy * split_d_inner * src0_nb1 + |
| 59 | + bidz * split_n_t * src0_nb0); |
| 60 | + const float * w_block = (const float *) ((char *) src1 + bidy * split_d_inner * src1_nb1); |
| 61 | + float * y_block = |
| 62 | + (float *) ((char *) dst + bidx * dst_nb2 + bidz * split_n_t * dst_nb1 + bidy * split_d_inner * dst_nb0); |
| 63 | + |
| 64 | + const int stride_x = src0_nb1 / sizeof(float); |
| 65 | + const int stride_w = src1_nb1 / sizeof(float); |
| 66 | + const int stride_y = dst_nb1 / sizeof(float); |
| 67 | + |
| 68 | + float x[d_conv] = { 0.0f }; |
| 69 | + float w[d_conv] = { 0.0f }; |
| 70 | + |
| 71 | +#pragma unroll |
| 72 | + for (int j = 0; j < d_conv; j++) { |
| 73 | + w[j] = w_block[tid * stride_w + j]; |
| 74 | + } |
| 75 | + |
| 76 | +#pragma unroll |
| 77 | + for (int i = 0; i < split_n_t; i++) { |
| 78 | + if (bidz * split_n_t + i < n_t) { |
| 79 | + float sumf = 0.0f; |
| 80 | + |
| 81 | + if (i == 0) { |
| 82 | + for (int j = 0; j < d_conv; j++) { |
| 83 | + x[j] = x_block[tid * stride_x + j]; |
| 84 | + } |
| 85 | + } else { |
| 86 | + x[(i - 1) % d_conv] = x_block[tid * stride_x + i + d_conv - 1]; |
| 87 | + } |
| 88 | + |
| 89 | +#pragma unroll |
| 90 | + for (int j = 0; j < d_conv; j++) { |
| 91 | + sumf += x[(i + j) % d_conv] * w[j]; |
| 92 | + } |
| 93 | + y_block[i * stride_y + tid] = sumf; |
| 94 | + } |
| 95 | + } |
| 96 | +} |
| 97 | + |
| 98 | +static void ssm_conv_f32_cuda(const float * src0, const float * src1, const int src0_nb0, const int src0_nb1, |
| 99 | + const int src0_nb2, const int src1_nb1, float * dst, const int dst_nb0, const int dst_nb1, |
| 100 | + const int dst_nb2, const int nc, const int ncs, const int nr, const int n_t, |
| 101 | + const int n_s, cudaStream_t stream) { |
| 102 | + const int threads = 128; |
| 103 | + GGML_ASSERT(nr % threads == 0); |
| 104 | + |
| 105 | + if (n_t <= 32) { |
| 106 | + const dim3 blocks(n_s, (nr + threads - 1) / threads, 1); |
| 107 | + if (nc == 4) { |
| 108 | + ssm_conv_f32<threads, 4><<<blocks, threads, 0, stream>>>(src0, src1, src0_nb0, src0_nb1, src0_nb2, src1_nb1, |
| 109 | + dst, dst_nb0, dst_nb1, dst_nb2, nc, ncs, nr, n_t, |
| 110 | + n_s); |
| 111 | + } else { |
| 112 | + GGML_ABORT("Only support kernel size = 4 now."); |
| 113 | + } |
| 114 | + } else { |
| 115 | + if (nc == 4) { |
| 116 | + const int split_n_t = 32; |
| 117 | + dim3 blocks(n_s, (nr + threads - 1) / threads, (n_t + split_n_t - 1) / split_n_t); |
| 118 | + ssm_conv_long_token_f32<threads, 4, split_n_t> |
| 119 | + <<<blocks, threads, 0, stream>>>(src0, src1, src0_nb0, src0_nb1, src0_nb2, src1_nb1, dst, dst_nb0, |
| 120 | + dst_nb1, dst_nb2, nc, ncs, nr, n_t, n_s); |
| 121 | + } else { |
| 122 | + GGML_ABORT("Only support kernel size = 4 right now."); |
| 123 | + } |
| 124 | + } |
| 125 | +} |
| 126 | + |
| 127 | +void ggml_cuda_op_ssm_conv(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { |
| 128 | + const struct ggml_tensor * src0 = dst->src[0]; // conv_x |
| 129 | + const struct ggml_tensor * src1 = dst->src[1]; // conv1d.weight |
| 130 | + |
| 131 | + const int nc = src1->ne[0]; // d_conv |
| 132 | + const int ncs = src0->ne[0]; // d_conv - 1 + n_t |
| 133 | + const int nr = src0->ne[1]; // d_inner |
| 134 | + const int n_t = dst->ne[1]; // tokens per sequence |
| 135 | + const int n_s = dst->ne[2]; // number of sequences in the batch |
| 136 | + |
| 137 | + GGML_ASSERT(dst->ne[0] == nr); |
| 138 | + GGML_ASSERT(src0->nb[0] == sizeof(float)); |
| 139 | + GGML_ASSERT(src1->nb[0] == sizeof(float)); |
| 140 | + GGML_ASSERT(src0->nb[1] == src0->ne[0] * sizeof(float)); |
| 141 | + |
| 142 | + const float * src0_d = (const float *) src0->data; |
| 143 | + const float * src1_d = (const float *) src1->data; |
| 144 | + float * dst_d = (float *) dst->data; |
| 145 | + cudaStream_t stream = ctx.stream(); |
| 146 | + |
| 147 | + GGML_ASSERT(src0->type == GGML_TYPE_F32); |
| 148 | + GGML_ASSERT(dst->type == GGML_TYPE_F32); |
| 149 | + ssm_conv_f32_cuda(src0_d, src1_d, src0->nb[0], src0->nb[1], src0->nb[2], src1->nb[1], dst_d, dst->nb[0], dst->nb[1], |
| 150 | + dst->nb[2], nc, ncs, nr, n_t, n_s, stream); |
| 151 | +} |
0 commit comments