compute/pkg/api/domain/domain.go

package domain

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"regexp"
	"time"

	"go.uber.org/zap"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	"libvirt.org/go/libvirt"

	"deevirt.fr/compute/pkg/api/proto"
	"deevirt.fr/compute/pkg/api/raft"
	"deevirt.fr/compute/pkg/config"
	"deevirt.fr/compute/pkg/scheduler"
)

type Domain struct {
	Config *config.Config
	Store  *raft.Store
	Logger *zap.Logger
	proto.UnimplementedDomainServer
}

func (d *Domain) connectNode(NodeId string) (*libvirt.Connect, error) {
	var jCluster raft.NodeStore
	cluster, _ := d.Store.Get("/etc/libvirt/cluster")
	json.Unmarshal(cluster, &jCluster)

	var libvirt_uri string
	if d.Config.LibvirtTLS {
		libvirt_uri = fmt.Sprintf("qemu+tls://%s/system", jCluster[NodeId].IpManagement)
	} else {
		libvirt_uri = fmt.Sprintf("qemu+tcp://%s/system", jCluster[NodeId].IpManagement)
	}

	c, err := libvirt.NewConnect(libvirt_uri)
	if err != nil {
		log.Fatalf("Erreur %v", err)
	}

	return c, nil
}

func (d *Domain) connectDomain(ctx context.Context, domainID string) (string, *libvirt.Connect, error) {
	dom, _ := d.Get(ctx, &proto.DomainListRequest{
		DomainId: domainID,
	})

	var jCluster raft.NodeStore
	cluster, _ := d.Store.Get("/etc/libvirt/cluster")
	json.Unmarshal(cluster, &jCluster)

	c, err := d.connectNode(dom.NodeId)

	return dom.NodeId, c, err
}

func (d *Domain) List(ctx context.Context, in *proto.DomainListAllRequest) (*proto.DomainListAllResponse, error) {
	test, _ := d.Store.Ls("/etc/libvirt/qemu", raft.LsOptions{
		Recursive: true,
		Data:      true,
	})

	domains := []*proto.DomainListResponse{}

	for k, v := range test {
		re := regexp.MustCompile("([^/]+)/([^/]+)")
		matches := re.FindStringSubmatch(k)
		if matches != nil {
			var dStore raft.DomainStore
			json.Unmarshal(v, &dStore)

			domains = append(domains, &proto.DomainListResponse{
				NodeId:   matches[1],
				DomainId: matches[2],
				Config:   dStore.Config,
				State:    int64(dStore.State),
			})
		}
	}

	return &proto.DomainListAllResponse{
		Domains: domains,
	}, nil
}

func (d *Domain) Get(ctx context.Context, in *proto.DomainListRequest) (*proto.DomainListResponse, error) {
	dom, _ := d.Store.Ls("/etc/libvirt/qemu", raft.LsOptions{
		Recursive: true,
		Data:      false,
	})

	for k := range dom {
		re := regexp.MustCompile("([^/]+)/([^/]+)")
		matches := re.FindStringSubmatch(k)
		if matches != nil && matches[2] == in.DomainId {
			var dStore raft.DomainStore
			data, _ := d.Store.Get(fmt.Sprintf("/etc/libvirt/qemu/%s/%s", matches[1], matches[2]))
			json.Unmarshal(data, &dStore)

			return &proto.DomainListResponse{
				NodeId:   matches[1],
				DomainId: matches[2],
				Config:   dStore.Config,
				State:    int64(dStore.State),
			}, nil
		}
	}

	return &proto.DomainListResponse{}, nil
}

func (d *Domain) Migrate(in *proto.DomainMigrateRequest, stream proto.Domain_MigrateServer) error {
	ctx := context.Background()

	nodeID, c, err := d.connectDomain(ctx, in.DomainId)
	if err != nil {
		return status.Errorf(codes.Internal, "Connexion error to libvirt")
	}
	defer c.Close()

	dom, err := c.LookupDomainByUUIDString(in.DomainId)
	if err != nil {
		return status.Errorf(codes.Internal, "Domain unknown")
	}

	s, err := scheduler.New()
	if err != nil {
		return status.Errorf(codes.Internal, "Connexion error to libvirt %v", err)
	}

	res, err := s.GetTopNode(1)
	if err != nil {
		return status.Errorf(codes.Internal, "Connexion error to libvirt %v", err)
	}

	ctx1, cancel := context.WithCancel(context.Background())
	ch := make(chan []byte)

	migrate := func(cancel context.CancelFunc) {
		defer cancel()

		c_new, err := d.connectNode(res[0].NodeID)
		if err != nil {
			d.Logger.Sugar().Infof("Connexion error to libvirt %v", err.Error())
			return
		}
		defer c_new.Close()

		new_dom, err := dom.Migrate(c_new, libvirt.MIGRATE_LIVE|libvirt.MIGRATE_PERSIST_DEST|libvirt.MIGRATE_UNDEFINE_SOURCE, "", "", 0)
		if err != nil {
			d.Logger.Sugar().Infof("Migration error %v", err.Error())
			return
		}

		newDomConfig, _ := new_dom.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
		newDomState, _, _ := new_dom.GetState()

		new, _ := json.Marshal(raft.DomainStore{
			Config:  newDomConfig,
			State:   int(newDomState),
			Migrate: false,
		})

		ch <- new
	}

	go migrate(cancel)

	for {
		select {
		case result := <-ch:
			d.Store.Set(fmt.Sprintf("/etc/libvirt/qemu/%s/%s", res[0].NodeID, in.DomainId), result)
			d.Store.Delete(fmt.Sprintf("/etc/libvirt/qemu/%s/%s", nodeID, in.DomainId))
		case <-ctx1.Done():
			return nil
		default:
			var queryMigrate struct {
				Return struct {
					RAM struct {
						Total     float64 `json:"total"`
						Remaining float64 `json:"remaining"`
					} `json:"ram"`
				} `json:"return"`
			}

			t, _ := dom.QemuMonitorCommand("{\"execute\": \"query-migrate\"}", libvirt.DOMAIN_QEMU_MONITOR_COMMAND_DEFAULT)
			if err := json.Unmarshal([]byte(t), &queryMigrate); err == nil {
				progress := (1 - (queryMigrate.Return.RAM.Remaining / queryMigrate.Return.RAM.Total)) * 100

				if progress > 0 {
					stream.Send(&proto.DomainMigrateResponse{
						Percentage: float32(progress),
					})

					d.Logger.Sugar().Infof("%s Progression: %.2f%%\n", nodeID, progress)
				}
			}

			time.Sleep(500 * time.Millisecond)
		}
	}
}